2024-11-10 03:13:33,073 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@dc7df28 2024-11-10 03:13:33,090 main DEBUG Took 0.012978 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-10 03:13:33,090 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-10 03:13:33,091 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-10 03:13:33,093 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-10 03:13:33,094 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 03:13:33,105 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-10 03:13:33,129 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 03:13:33,131 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 03:13:33,132 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 03:13:33,133 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 03:13:33,133 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 03:13:33,134 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 03:13:33,138 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 03:13:33,138 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 03:13:33,139 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 03:13:33,139 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 03:13:33,142 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 03:13:33,148 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 03:13:33,149 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 03:13:33,150 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 03:13:33,151 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 03:13:33,151 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 03:13:33,152 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 03:13:33,152 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 03:13:33,153 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 03:13:33,153 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 03:13:33,154 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 03:13:33,155 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 03:13:33,156 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 03:13:33,157 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 03:13:33,157 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 03:13:33,158 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-10 03:13:33,161 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 03:13:33,164 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-10 03:13:33,166 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-10 03:13:33,167 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-10 03:13:33,169 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-10 03:13:33,169 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-10 03:13:33,180 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-10 03:13:33,184 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-10 03:13:33,187 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-10 03:13:33,187 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-10 03:13:33,188 main DEBUG createAppenders(={Console}) 2024-11-10 03:13:33,189 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@dc7df28 initialized 2024-11-10 03:13:33,190 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@dc7df28 2024-11-10 03:13:33,190 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@dc7df28 OK. 2024-11-10 03:13:33,191 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-10 03:13:33,191 main DEBUG OutputStream closed 2024-11-10 03:13:33,192 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-10 03:13:33,192 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-10 03:13:33,192 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@316bcf94 OK 2024-11-10 03:13:33,318 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-10 03:13:33,321 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-10 03:13:33,322 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-10 03:13:33,323 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-10 03:13:33,324 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-10 03:13:33,324 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-10 03:13:33,325 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-10 03:13:33,325 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-10 03:13:33,326 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-10 03:13:33,326 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-10 03:13:33,327 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-10 03:13:33,327 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-10 03:13:33,327 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-10 03:13:33,328 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-10 03:13:33,328 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-10 03:13:33,329 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-10 03:13:33,329 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-10 03:13:33,330 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-10 03:13:33,333 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10 03:13:33,334 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19e4fcac) with optional ClassLoader: null 2024-11-10 03:13:33,334 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-10 03:13:33,335 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19e4fcac] started OK. 2024-11-10T03:13:33,352 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.janitor.TestCatalogJanitor timeout: 13 mins 2024-11-10 03:13:33,357 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-10 03:13:33,358 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10T03:13:33,752 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79 2024-11-10T03:13:33,797 INFO [Time-limited test {}] regionserver.ChunkCreator(472): data poolSizePercentage is less than 0. So not using pool 2024-11-10T03:13:33,797 INFO [Time-limited test {}] regionserver.ChunkCreator(472): index poolSizePercentage is less than 0. So not using pool 2024-11-10T03:13:33,817 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.janitor.TestCatalogJanitor#testCleanParent Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=956, ProcessCount=11, AvailableMemoryMB=2279 2024-11-10T03:13:33,958 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-10T03:13:35,388 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T03:13:35,438 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-10T03:13:35,595 INFO [Time-limited test {}] util.FSUtils(489): Created version file at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanParent with version=8 2024-11-10T03:13:35,617 DEBUG [Time-limited test {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanParent/hbase.id] with ID: 9beab228-1349-4508-82d6-e10e1292200b 2024-11-10T03:13:35,618 DEBUG [Time-limited test {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanParent/.tmp/hbase.id 2024-11-10T03:13:35,628 DEBUG [Time-limited test {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanParent/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanParent/hbase.id] 2024-11-10T03:13:35,679 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T03:13:35,708 INFO [Time-limited test {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T03:13:35,712 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T03:13:35,813 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.MockMasterServices.(MockMasterServices.java:119) ~[test-classes/:?] at org.apache.hadoop.hbase.master.janitor.TestCatalogJanitor.setup(TestCatalogJanitor.java:110) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.invokeMethod(RunBefores.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:13:35,819 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T03:13:35,825 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-10T03:13:35,838 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-10T03:13:35,838 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-10T03:13:35,916 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-10T03:13:35,924 INFO [Time-limited test {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanParent/MasterData/data/master/store 2024-11-10T03:13:35,955 INFO [Time-limited test {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-10T03:13:35,961 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:13:36,037 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T03:13:36,038 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:36,039 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:36,041 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T03:13:36,042 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:36,042 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:36,044 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731208416037Disabling compacts and flushes for region at 1731208416037Disabling writes for close at 1731208416042 (+5 ms)Writing region close event to WAL at 1731208416042Closed at 1731208416042 2024-11-10T03:13:36,045 WARN [Time-limited test {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanParent/MasterData/data/master/store/.initializing 2024-11-10T03:13:36,046 DEBUG [Time-limited test {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanParent/MasterData/WALs/mockmaster.example.org,1234,-1 2024-11-10T03:13:36,081 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T03:13:36,117 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=mockmaster.example.org%2C1234%2C-1, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanParent/MasterData/WALs/mockmaster.example.org,1234,-1, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanParent/MasterData/oldWALs, maxLogs=10 2024-11-10T03:13:36,292 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanParent/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731208416207 2024-11-10T03:13:36,297 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T03:13:36,299 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T03:13:36,300 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:13:36,306 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:36,308 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:36,368 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:36,415 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T03:13:36,421 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:36,427 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T03:13:36,428 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:36,433 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T03:13:36,433 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:36,434 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:13:36,434 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:36,438 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T03:13:36,439 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:36,440 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:13:36,440 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:36,446 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T03:13:36,446 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:36,447 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:13:36,449 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:36,455 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanParent/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:36,456 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanParent/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:36,460 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:36,461 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:36,466 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T03:13:36,470 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:36,486 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanParent/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T03:13:36,487 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58851517, jitterRate=-0.12304405868053436}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T03:13:36,496 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731208416328Initializing all the Stores at 1731208416329 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731208416329Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208416333 (+4 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208416333Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208416333Cleaning up temporary data from old regions at 1731208416462 (+129 ms)Region opened successfully at 1731208416496 (+34 ms) 2024-11-10T03:13:36,498 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T03:13:36,800 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T03:13:36,805 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered NoopProcedureStore lease in 0 msec 2024-11-10T03:13:36,807 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded NoopProcedureStore in 0 msec 2024-11-10T03:13:36,811 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731208446811 2024-11-10T03:13:36,811 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T03:13:36,855 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,100,1 2024-11-10T03:13:36,858 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,101,1 2024-11-10T03:13:36,859 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,102,1 2024-11-10T03:13:36,859 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,103,1 2024-11-10T03:13:36,860 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,104,1 2024-11-10T03:13:36,860 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,105,1 2024-11-10T03:13:36,860 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,106,1 2024-11-10T03:13:36,861 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,107,1 2024-11-10T03:13:36,861 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,108,1 2024-11-10T03:13:36,861 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,109,1 2024-11-10T03:13:36,871 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:36,900 INFO [Time-limited test {}] janitor.TestCatalogJanitor(270): Created reference file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanParent/data/default/testCleanParent/3e534d5e05d1f67d9a6b09a404e8b6ac/cf/1731208416870.3e534d5e05d1f67d9a6b09a404e8b6ac 2024-11-10T03:13:36,901 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => 3e534d5e05d1f67d9a6b09a404e8b6ac, NAME => 'testCleanParent,aaa,1731208416864.3e534d5e05d1f67d9a6b09a404e8b6ac.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T03:13:36,911 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:36,918 DEBUG [Time-limited test {}] janitor.CatalogJanitor(364): Deferring removal of region {ENCODED => 3e534d5e05d1f67d9a6b09a404e8b6ac, NAME => 'testCleanParent,aaa,1731208416864.3e534d5e05d1f67d9a6b09a404e8b6ac.', STARTKEY => 'aaa', ENDKEY => 'eee'} because daughter {ENCODED => 3e534d5e05d1f67d9a6b09a404e8b6ac, NAME => 'testCleanParent,aaa,1731208416864.3e534d5e05d1f67d9a6b09a404e8b6ac.', STARTKEY => 'aaa', ENDKEY => 'ccc'} still has references 2024-11-10T03:13:36,919 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => 3e534d5e05d1f67d9a6b09a404e8b6ac, NAME => 'testCleanParent,aaa,1731208416864.3e534d5e05d1f67d9a6b09a404e8b6ac.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T03:13:36,921 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:36,921 DEBUG [Time-limited test {}] janitor.CatalogJanitor(351): Deleting region 3e534d5e05d1f67d9a6b09a404e8b6ac because daughters -- 3e534d5e05d1f67d9a6b09a404e8b6ac, dc7ce0cc315b3b0b1c07e016bae2f670 -- no longer hold references 2024-11-10T03:13:36,923 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testCleanParent, region=3e534d5e05d1f67d9a6b09a404e8b6ac 2024-11-10T03:13:36,932 DEBUG [Time-limited test {}] janitor.CatalogJanitor(358): Submitted procedure pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testCleanParent, region=3e534d5e05d1f67d9a6b09a404e8b6ac for split parent {ENCODED => 3e534d5e05d1f67d9a6b09a404e8b6ac, NAME => 'testCleanParent,aaa,1731208416864.3e534d5e05d1f67d9a6b09a404e8b6ac.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T03:13:36,933 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testCleanParent, region=3e534d5e05d1f67d9a6b09a404e8b6ac 2024-11-10T03:13:36,937 DEBUG [PEWorker-1 {}] assignment.GCRegionProcedure(87): Archiving region=3e534d5e05d1f67d9a6b09a404e8b6ac 2024-11-10T03:13:36,938 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanParent/data/default/testCleanParent/3e534d5e05d1f67d9a6b09a404e8b6ac 2024-11-10T03:13:36,941 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanParent/data/default/testCleanParent/3e534d5e05d1f67d9a6b09a404e8b6ac/cf] 2024-11-10T03:13:36,958 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanParent/data/default/testCleanParent/3e534d5e05d1f67d9a6b09a404e8b6ac 2024-11-10T03:13:36,973 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testCleanParent,aaa,1731208416864.3e534d5e05d1f67d9a6b09a404e8b6ac.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731208416959"}]},"ts":"9223372036854775807"} 2024-11-10T03:13:36,974 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T03:13:36,974 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3e534d5e05d1f67d9a6b09a404e8b6ac, NAME => 'testCleanParent,aaa,1731208416864.3e534d5e05d1f67d9a6b09a404e8b6ac.', STARTKEY => 'aaa', ENDKEY => 'eee'}] 2024-11-10T03:13:36,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; GCRegionProcedure table=testCleanParent, region=3e534d5e05d1f67d9a6b09a404e8b6ac in 51 msec 2024-11-10T03:13:37,183 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T03:13:37,184 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T03:13:37,184 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T03:13:37,192 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.janitor.TestCatalogJanitor#testCleanParent Thread=24 (was 12) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: master:store-Flusher java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1759) app//org.apache.hadoop.hbase.master.region.MasterRegionFlusherAndCompactor.flushLoop(MasterRegionFlusherAndCompactor.java:200) app//org.apache.hadoop.hbase.master.region.MasterRegionFlusherAndCompactor$$Lambda$299/0x00007fe9644f6890.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: AsyncFSWAL-0-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanParent/MasterData-prefix:mockmaster.example.org,1234,-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSOutputFlusher-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanParent/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731208416207 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master:store-WAL-Roller java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:180) - Thread LEAK? -, OpenFileDescriptor=321 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=967 (was 956) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2104 (was 2279) 2024-11-10T03:13:37,199 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.janitor.TestCatalogJanitor#testCleanMerge Thread=24, OpenFileDescriptor=321, MaxFileDescriptor=1048576, SystemLoadAverage=967, ProcessCount=11, AvailableMemoryMB=2109 2024-11-10T03:13:37,219 INFO [Time-limited test {}] util.FSUtils(489): Created version file at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanMerge with version=8 2024-11-10T03:13:37,220 DEBUG [Time-limited test {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanMerge/hbase.id] with ID: 10250486-a8e3-4496-a699-f044fafbe766 2024-11-10T03:13:37,220 DEBUG [Time-limited test {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanMerge/.tmp/hbase.id 2024-11-10T03:13:37,227 DEBUG [Time-limited test {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanMerge/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanMerge/hbase.id] 2024-11-10T03:13:37,285 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T03:13:37,286 INFO [Time-limited test {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T03:13:37,287 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T03:13:37,288 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T03:13:37,315 INFO [Time-limited test {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanMerge/MasterData/data/master/store 2024-11-10T03:13:37,326 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:13:37,326 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T03:13:37,326 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:37,326 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:37,326 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T03:13:37,326 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:37,327 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:37,327 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731208417326Disabling compacts and flushes for region at 1731208417326Disabling writes for close at 1731208417326Writing region close event to WAL at 1731208417327 (+1 ms)Closed at 1731208417327 2024-11-10T03:13:37,327 WARN [Time-limited test {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanMerge/MasterData/data/master/store/.initializing 2024-11-10T03:13:37,327 DEBUG [Time-limited test {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanMerge/MasterData/WALs/mockmaster.example.org,1234,-1 2024-11-10T03:13:37,335 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T03:13:37,339 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=mockmaster.example.org%2C1234%2C-1, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanMerge/MasterData/WALs/mockmaster.example.org,1234,-1, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanMerge/MasterData/oldWALs, maxLogs=10 2024-11-10T03:13:37,368 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanMerge/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731208417340 2024-11-10T03:13:37,369 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T03:13:37,369 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T03:13:37,370 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:13:37,370 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:37,370 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:37,372 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:37,379 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T03:13:37,380 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:37,380 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T03:13:37,381 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:37,385 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T03:13:37,385 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:37,386 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:13:37,386 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:37,394 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T03:13:37,395 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:37,395 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:13:37,396 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:37,401 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T03:13:37,401 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:37,402 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:13:37,402 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:37,404 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanMerge/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:37,404 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanMerge/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:37,405 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:37,405 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:37,406 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T03:13:37,407 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:37,428 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanMerge/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T03:13:37,429 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66027439, jitterRate=-0.0161144882440567}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T03:13:37,429 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731208417371Initializing all the Stores at 1731208417372 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731208417372Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208417372Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208417372Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208417372Cleaning up temporary data from old regions at 1731208417405 (+33 ms)Region opened successfully at 1731208417429 (+24 ms) 2024-11-10T03:13:37,430 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T03:13:37,444 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T03:13:37,445 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered NoopProcedureStore lease in 0 msec 2024-11-10T03:13:37,445 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded NoopProcedureStore in 0 msec 2024-11-10T03:13:37,453 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731208447453 2024-11-10T03:13:37,453 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T03:13:37,455 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,100,1 2024-11-10T03:13:37,455 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,101,1 2024-11-10T03:13:37,456 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,102,1 2024-11-10T03:13:37,456 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,103,1 2024-11-10T03:13:37,457 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,104,1 2024-11-10T03:13:37,458 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,105,1 2024-11-10T03:13:37,458 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,106,1 2024-11-10T03:13:37,459 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,107,1 2024-11-10T03:13:37,459 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,108,1 2024-11-10T03:13:37,459 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,109,1 2024-11-10T03:13:37,461 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:37,510 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:37,522 DEBUG [Time-limited test {}] janitor.CatalogJanitor(258): Cleaning merged region {ENCODED => 68b5131ee9b93170c08f60f8d044d408, NAME => 'testCleanMerge,aaa,1731208417460.68b5131ee9b93170c08f60f8d044d408.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T03:13:37,522 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:37,524 DEBUG [Time-limited test {}] janitor.CatalogJanitor(283): Deferring cleanup up of 2 parents of merged region 68b5131ee9b93170c08f60f8d044d408, because references still exist in merged region or we encountered an exception in checking 2024-11-10T03:13:37,524 DEBUG [Time-limited test {}] janitor.CatalogJanitor(258): Cleaning merged region {ENCODED => 68b5131ee9b93170c08f60f8d044d408, NAME => 'testCleanMerge,aaa,1731208417460.68b5131ee9b93170c08f60f8d044d408.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T03:13:37,525 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:37,526 DEBUG [Time-limited test {}] janitor.CatalogJanitor(283): Deferring cleanup up of 2 parents of merged region 68b5131ee9b93170c08f60f8d044d408, because references still exist in merged region or we encountered an exception in checking 2024-11-10T03:13:37,526 DEBUG [Time-limited test {}] janitor.CatalogJanitor(258): Cleaning merged region {ENCODED => 68b5131ee9b93170c08f60f8d044d408, NAME => 'testCleanMerge,aaa,1731208417460.68b5131ee9b93170c08f60f8d044d408.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T03:13:37,526 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:37,527 DEBUG [Time-limited test {}] janitor.CatalogJanitor(266): Deleting parents ([68b5131ee9b93170c08f60f8d044d408], [2dd0eb0eeada14856ad5d74f0e2cd19c]) from fs; merged child {ENCODED => 68b5131ee9b93170c08f60f8d044d408, NAME => 'testCleanMerge,aaa,1731208417460.68b5131ee9b93170c08f60f8d044d408.', STARTKEY => 'aaa', ENDKEY => 'eee'} no longer holds references 2024-11-10T03:13:37,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:GC_MERGED_REGIONS_PREPARE, hasLock=false; GCMultipleMergedRegionsProcedure child=68b5131ee9b93170c08f60f8d044d408, parents:[68b5131ee9b93170c08f60f8d044d408], [2dd0eb0eeada14856ad5d74f0e2cd19c] 2024-11-10T03:13:37,529 DEBUG [Time-limited test {}] janitor.CatalogJanitor(277): Submitted procedure pid=1, state=RUNNABLE:GC_MERGED_REGIONS_PREPARE, hasLock=false; GCMultipleMergedRegionsProcedure child=68b5131ee9b93170c08f60f8d044d408, parents:[68b5131ee9b93170c08f60f8d044d408], [2dd0eb0eeada14856ad5d74f0e2cd19c] for merged region {ENCODED => 68b5131ee9b93170c08f60f8d044d408, NAME => 'testCleanMerge,aaa,1731208417460.68b5131ee9b93170c08f60f8d044d408.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T03:13:37,530 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T03:13:37,529 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=1, state=RUNNABLE:GC_MERGED_REGIONS_PREPARE, hasLock=false; GCMultipleMergedRegionsProcedure child=68b5131ee9b93170c08f60f8d044d408, parents:[68b5131ee9b93170c08f60f8d044d408], [2dd0eb0eeada14856ad5d74f0e2cd19c] 2024-11-10T03:13:37,530 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T03:13:37,530 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T03:13:37,532 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=1, state=RUNNABLE:GC_MERGED_REGIONS_PREPARE, hasLock=true; GCMultipleMergedRegionsProcedure child=68b5131ee9b93170c08f60f8d044d408, parents:[68b5131ee9b93170c08f60f8d044d408], [2dd0eb0eeada14856ad5d74f0e2cd19c] java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hbase.client.Result.rawCells()" because the return value of "org.apache.hadoop.hbase.master.assignment.RegionStateStore.getRegionCatalogResult(org.apache.hadoop.hbase.client.RegionInfo)" is null at org.apache.hadoop.hbase.master.assignment.RegionStateStore.getMergeRegions(RegionStateStore.java:474) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.GCMultipleMergedRegionsProcedure.executeFromState(GCMultipleMergedRegionsProcedure.java:103) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.GCMultipleMergedRegionsProcedure.executeFromState(GCMultipleMergedRegionsProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-10T03:13:37,534 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=1, state=FAILED:GC_MERGED_REGIONS_PREPARE, hasLock=true, exception=java.lang.NullPointerException via CODE-BUG: Uncaught runtime exception: pid=1, state=RUNNABLE:GC_MERGED_REGIONS_PREPARE, hasLock=true; GCMultipleMergedRegionsProcedure child=68b5131ee9b93170c08f60f8d044d408, parents:[68b5131ee9b93170c08f60f8d044d408], [2dd0eb0eeada14856ad5d74f0e2cd19c]:java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hbase.client.Result.rawCells()" because the return value of "org.apache.hadoop.hbase.master.assignment.RegionStateStore.getRegionCatalogResult(org.apache.hadoop.hbase.client.RegionInfo)" is null; GCMultipleMergedRegionsProcedure child=68b5131ee9b93170c08f60f8d044d408, parents:[68b5131ee9b93170c08f60f8d044d408], [2dd0eb0eeada14856ad5d74f0e2cd19c] does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hbase.client.Result.rawCells()" because the return value of "org.apache.hadoop.hbase.master.assignment.RegionStateStore.getRegionCatalogResult(org.apache.hadoop.hbase.client.RegionInfo)" is null at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hbase.client.Result.rawCells()" because the return value of "org.apache.hadoop.hbase.master.assignment.RegionStateStore.getRegionCatalogResult(org.apache.hadoop.hbase.client.RegionInfo)" is null at org.apache.hadoop.hbase.master.assignment.RegionStateStore.getMergeRegions(RegionStateStore.java:474) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.GCMultipleMergedRegionsProcedure.executeFromState(GCMultipleMergedRegionsProcedure.java:103) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.GCMultipleMergedRegionsProcedure.executeFromState(GCMultipleMergedRegionsProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-11-10T03:13:37,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=1, state=ROLLEDBACK, hasLock=true, exception=java.lang.NullPointerException via CODE-BUG: Uncaught runtime exception: pid=1, state=RUNNABLE:GC_MERGED_REGIONS_PREPARE, hasLock=true; GCMultipleMergedRegionsProcedure child=68b5131ee9b93170c08f60f8d044d408, parents:[68b5131ee9b93170c08f60f8d044d408], [2dd0eb0eeada14856ad5d74f0e2cd19c]:java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hbase.client.Result.rawCells()" because the return value of "org.apache.hadoop.hbase.master.assignment.RegionStateStore.getRegionCatalogResult(org.apache.hadoop.hbase.client.RegionInfo)" is null; GCMultipleMergedRegionsProcedure child=68b5131ee9b93170c08f60f8d044d408, parents:[68b5131ee9b93170c08f60f8d044d408], [2dd0eb0eeada14856ad5d74f0e2cd19c] exec-time=7 msec 2024-11-10T03:13:37,536 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-10T03:13:37,541 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.janitor.TestCatalogJanitor#testCleanMerge Thread=31 (was 24) Potentially hanging thread: PEWorker-1 java.base@17.0.11/java.lang.ref.Reference.refersTo0(Native Method) java.base@17.0.11/java.lang.ref.Reference.refersToImpl(Reference.java:375) java.base@17.0.11/java.lang.ref.Reference.refersTo(Reference.java:366) java.base@17.0.11/java.lang.reflect.AccessibleObject.isAccessChecked(AccessibleObject.java:664) java.base@17.0.11/java.lang.reflect.AccessibleObject.verifyAccess(AccessibleObject.java:695) java.base@17.0.11/java.lang.reflect.AccessibleObject.checkAccess(AccessibleObject.java:673) java.base@17.0.11/java.lang.reflect.Method.invoke(Method.java:560) app//org.mockito.internal.debugging.Java9PlusLocationImpl$MetadataShim.get(Java9PlusLocationImpl.java:257) app//org.mockito.internal.debugging.Java9PlusLocationImpl$MetadataShim.getClassName(Java9PlusLocationImpl.java:216) app//org.mockito.internal.exceptions.stacktrace.DefaultStackTraceCleaner.isIn(DefaultStackTraceCleaner.java:21) app//org.mockito.internal.debugging.Java9PlusLocationImpl$$Lambda$301/0x00007fe96450c240.test(Unknown Source) java.base@17.0.11/java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:178) java.base@17.0.11/java.util.stream.SliceOps$1$1.accept(SliceOps.java:200) java.base@17.0.11/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:197) java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.tryAdvance(StackStreamFactory.java:595) java.base@17.0.11/java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:129) java.base@17.0.11/java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:527) java.base@17.0.11/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:513) java.base@17.0.11/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499) java.base@17.0.11/java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:150) java.base@17.0.11/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) java.base@17.0.11/java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:647) app//org.mockito.internal.debugging.Java9PlusLocationImpl.lambda$getStackFrame$2(Java9PlusLocationImpl.java:107) app//org.mockito.internal.debugging.Java9PlusLocationImpl$$Lambda$304/0x00007fe96450cb58.apply(Unknown Source) java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.consumeFrames(StackStreamFactory.java:534) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.doStackWalk(StackStreamFactory.java:306) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.callStackWalk(Native Method) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.beginStackWalk(StackStreamFactory.java:370) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.walk(StackStreamFactory.java:243) java.base@17.0.11/java.lang.StackWalker.walk(StackWalker.java:499) java.base@17.0.11/jdk.internal.reflect.GeneratedMethodAccessor32.invoke(Unknown Source) java.base@17.0.11/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.base@17.0.11/java.lang.reflect.Method.invoke(Method.java:568) app//org.mockito.internal.debugging.Java9PlusLocationImpl.stackWalk(Java9PlusLocationImpl.java:147) app//org.mockito.internal.debugging.Java9PlusLocationImpl.getStackFrame(Java9PlusLocationImpl.java:101) app//org.mockito.internal.debugging.Java9PlusLocationImpl.(Java9PlusLocationImpl.java:76) app//org.mockito.internal.debugging.LocationFactory$Java9PlusLocationFactory.create(LocationFactory.java:46) app//org.mockito.internal.debugging.LocationFactory.create(LocationFactory.java:19) app//org.mockito.internal.debugging.LocationFactory.create(LocationFactory.java:15) app//org.mockito.internal.creation.bytebuddy.MockMethodInterceptor.doIntercept(MockMethodInterceptor.java:56) app//org.mockito.internal.creation.bytebuddy.MockMethodInterceptor$DispatcherDefaultingToRealMethod.interceptSuperCallable(MockMethodInterceptor.java:145) app//org.apache.hadoop.hbase.client.Connection$MockitoMock$KhgVvZZC.getTable(Unknown Source) app//org.apache.hadoop.hbase.master.assignment.RegionStateStore.getMetaTable(RegionStateStore.java:341) app//org.apache.hadoop.hbase.master.assignment.RegionStateStore.getRegionCatalogResult(RegionStateStore.java:347) app//org.apache.hadoop.hbase.master.assignment.RegionStateStore.getMergeRegions(RegionStateStore.java:474) app//org.apache.hadoop.hbase.master.assignment.GCMultipleMergedRegionsProcedure.executeFromState(GCMultipleMergedRegionsProcedure.java:103) app//org.apache.hadoop.hbase.master.assignment.GCMultipleMergedRegionsProcedure.executeFromState(GCMultipleMergedRegionsProcedure.java:45) app//org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) app//org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread$$Lambda$329/0x00007fe96455e6a8.call(Unknown Source) app//org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) Potentially hanging thread: ProcedureDispatcherTimeoutThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.procedure2.util.DelayedUtil.takeWithoutInterrupt(DelayedUtil.java:81) app//org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher$TimeoutExecutorThread.run(RemoteProcedureDispatcher.java:328) Potentially hanging thread: AsyncFSWAL-0-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanMerge/MasterData-prefix:mockmaster.example.org,1234,-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: AsyncFSOutputFlusher-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testCleanMerge/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731208417340 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=355 (was 321) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=967 (was 967), ProcessCount=11 (was 11), AvailableMemoryMB=2087 (was 2109) 2024-11-10T03:13:37,550 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.janitor.TestCatalogJanitor#testScanDoesNotCleanRegionsWithExistingParents Thread=29, OpenFileDescriptor=355, MaxFileDescriptor=1048576, SystemLoadAverage=967, ProcessCount=11, AvailableMemoryMB=2086 2024-11-10T03:13:37,579 INFO [Time-limited test {}] util.FSUtils(489): Created version file at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testScanDoesNotCleanRegionsWithExistingParents with version=8 2024-11-10T03:13:37,580 DEBUG [Time-limited test {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testScanDoesNotCleanRegionsWithExistingParents/hbase.id] with ID: a3debec8-4345-4c1f-80f9-300b07406d27 2024-11-10T03:13:37,580 DEBUG [Time-limited test {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testScanDoesNotCleanRegionsWithExistingParents/.tmp/hbase.id 2024-11-10T03:13:37,596 DEBUG [Time-limited test {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testScanDoesNotCleanRegionsWithExistingParents/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testScanDoesNotCleanRegionsWithExistingParents/hbase.id] 2024-11-10T03:13:37,682 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T03:13:37,683 INFO [Time-limited test {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T03:13:37,686 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T03:13:37,687 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T03:13:37,755 INFO [Time-limited test {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testScanDoesNotCleanRegionsWithExistingParents/MasterData/data/master/store 2024-11-10T03:13:37,783 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:13:37,784 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T03:13:37,784 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:37,785 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:37,785 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T03:13:37,785 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:37,785 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:37,786 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731208417784Disabling compacts and flushes for region at 1731208417784Disabling writes for close at 1731208417785 (+1 ms)Writing region close event to WAL at 1731208417785Closed at 1731208417785 2024-11-10T03:13:37,786 WARN [Time-limited test {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testScanDoesNotCleanRegionsWithExistingParents/MasterData/data/master/store/.initializing 2024-11-10T03:13:37,786 DEBUG [Time-limited test {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testScanDoesNotCleanRegionsWithExistingParents/MasterData/WALs/mockmaster.example.org,1234,-1 2024-11-10T03:13:37,801 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T03:13:37,811 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=mockmaster.example.org%2C1234%2C-1, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testScanDoesNotCleanRegionsWithExistingParents/MasterData/WALs/mockmaster.example.org,1234,-1, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testScanDoesNotCleanRegionsWithExistingParents/MasterData/oldWALs, maxLogs=10 2024-11-10T03:13:37,853 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testScanDoesNotCleanRegionsWithExistingParents/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731208417813 2024-11-10T03:13:37,854 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T03:13:37,854 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T03:13:37,855 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:13:37,855 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:37,855 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:37,861 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:37,865 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T03:13:37,866 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:37,866 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T03:13:37,866 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:37,873 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T03:13:37,873 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:37,874 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:13:37,874 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:37,886 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T03:13:37,887 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:37,887 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:13:37,887 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:37,896 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T03:13:37,896 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:37,897 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:13:37,898 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:37,900 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testScanDoesNotCleanRegionsWithExistingParents/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:37,900 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testScanDoesNotCleanRegionsWithExistingParents/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:37,901 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:37,901 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:37,902 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T03:13:37,905 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:37,941 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testScanDoesNotCleanRegionsWithExistingParents/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T03:13:37,942 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63864045, jitterRate=-0.04835157096385956}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T03:13:37,942 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731208417856Initializing all the Stores at 1731208417856Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731208417856Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208417860 (+4 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208417860Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208417860Cleaning up temporary data from old regions at 1731208417902 (+42 ms)Region opened successfully at 1731208417942 (+40 ms) 2024-11-10T03:13:37,945 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T03:13:37,975 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T03:13:37,976 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered NoopProcedureStore lease in 0 msec 2024-11-10T03:13:37,976 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded NoopProcedureStore in 0 msec 2024-11-10T03:13:38,012 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731208448012 2024-11-10T03:13:38,013 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T03:13:38,037 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,100,1 2024-11-10T03:13:38,038 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,101,1 2024-11-10T03:13:38,038 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,102,1 2024-11-10T03:13:38,039 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,103,1 2024-11-10T03:13:38,039 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,104,1 2024-11-10T03:13:38,041 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,105,1 2024-11-10T03:13:38,041 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,106,1 2024-11-10T03:13:38,042 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,107,1 2024-11-10T03:13:38,043 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,108,1 2024-11-10T03:13:38,043 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,109,1 2024-11-10T03:13:41,077 INFO [Time-limited test {}] janitor.TestCatalogJanitor(451): parent=09efdeef9bad77e33c2f27b176a37faf, splita=662b2b4ec89923845e6b3b779abe59af 2024-11-10T03:13:41,077 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:41,092 INFO [Time-limited test {}] janitor.TestCatalogJanitor(454): Created reference file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testScanDoesNotCleanRegionsWithExistingParents/data/default/testScanDoesNotCleanRegionsWithExistingParents/662b2b4ec89923845e6b3b779abe59af/cf/1731208421077.09efdeef9bad77e33c2f27b176a37faf 2024-11-10T03:13:41,092 DEBUG [Time-limited test {}] janitor.CatalogJanitor(180): 2024-11-10T03:13:41,092 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => 09efdeef9bad77e33c2f27b176a37faf, NAME => 'testScanDoesNotCleanRegionsWithExistingParents,aaa,1731208418043.09efdeef9bad77e33c2f27b176a37faf.', STARTKEY => 'aaa', ENDKEY => '', SPLIT => true} 2024-11-10T03:13:41,094 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:41,094 DEBUG [Time-limited test {}] janitor.CatalogJanitor(364): Deferring removal of region {ENCODED => 09efdeef9bad77e33c2f27b176a37faf, NAME => 'testScanDoesNotCleanRegionsWithExistingParents,aaa,1731208418043.09efdeef9bad77e33c2f27b176a37faf.', STARTKEY => 'aaa', ENDKEY => '', SPLIT => true} because daughter {ENCODED => 662b2b4ec89923845e6b3b779abe59af, NAME => 'testScanDoesNotCleanRegionsWithExistingParents,aaa,1731208419045.662b2b4ec89923845e6b3b779abe59af.', STARTKEY => 'aaa', ENDKEY => 'ccc', SPLIT => true} still has references 2024-11-10T03:13:41,096 DEBUG [Time-limited test {}] janitor.CatalogJanitor(180): 2024-11-10T03:13:41,096 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => 09efdeef9bad77e33c2f27b176a37faf, NAME => 'testScanDoesNotCleanRegionsWithExistingParents,aaa,1731208418043.09efdeef9bad77e33c2f27b176a37faf.', STARTKEY => 'aaa', ENDKEY => '', SPLIT => true} 2024-11-10T03:13:41,097 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:41,097 DEBUG [Time-limited test {}] janitor.CatalogJanitor(351): Deleting region 09efdeef9bad77e33c2f27b176a37faf because daughters -- 662b2b4ec89923845e6b3b779abe59af, 7c64cc07fe8780c45da833bb3a41bcc9 -- no longer hold references 2024-11-10T03:13:41,097 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testScanDoesNotCleanRegionsWithExistingParents, region=09efdeef9bad77e33c2f27b176a37faf 2024-11-10T03:13:41,098 DEBUG [Time-limited test {}] janitor.CatalogJanitor(358): Submitted procedure pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testScanDoesNotCleanRegionsWithExistingParents, region=09efdeef9bad77e33c2f27b176a37faf for split parent {ENCODED => 09efdeef9bad77e33c2f27b176a37faf, NAME => 'testScanDoesNotCleanRegionsWithExistingParents,aaa,1731208418043.09efdeef9bad77e33c2f27b176a37faf.', STARTKEY => 'aaa', ENDKEY => '', SPLIT => true} 2024-11-10T03:13:41,098 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => 662b2b4ec89923845e6b3b779abe59af, NAME => 'testScanDoesNotCleanRegionsWithExistingParents,aaa,1731208419045.662b2b4ec89923845e6b3b779abe59af.', STARTKEY => 'aaa', ENDKEY => 'ccc', OFFLINE => true, SPLIT => true} 2024-11-10T03:13:41,098 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testScanDoesNotCleanRegionsWithExistingParents, region=09efdeef9bad77e33c2f27b176a37faf 2024-11-10T03:13:41,099 DEBUG [Time-limited test {}] janitor.CatalogJanitor(351): Deleting region 662b2b4ec89923845e6b3b779abe59af because daughters -- f2527e88a530de42d8c6c7c747d4b0d0, 6e9a779fd8dc39cba385c3b7e63c68b7 -- no longer hold references 2024-11-10T03:13:41,099 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=2, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testScanDoesNotCleanRegionsWithExistingParents, region=662b2b4ec89923845e6b3b779abe59af 2024-11-10T03:13:41,099 DEBUG [Time-limited test {}] janitor.CatalogJanitor(358): Submitted procedure pid=2, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testScanDoesNotCleanRegionsWithExistingParents, region=662b2b4ec89923845e6b3b779abe59af for split parent {ENCODED => 662b2b4ec89923845e6b3b779abe59af, NAME => 'testScanDoesNotCleanRegionsWithExistingParents,aaa,1731208419045.662b2b4ec89923845e6b3b779abe59af.', STARTKEY => 'aaa', ENDKEY => 'ccc', OFFLINE => true, SPLIT => true} 2024-11-10T03:13:41,099 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testScanDoesNotCleanRegionsWithExistingParents, region=662b2b4ec89923845e6b3b779abe59af 2024-11-10T03:13:41,099 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T03:13:41,099 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testScanDoesNotCleanRegionsWithExistingParents,aaa,1731208418043.09efdeef9bad77e33c2f27b176a37faf.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731208421099"}]},"ts":"9223372036854775807"} 2024-11-10T03:13:41,099 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T03:13:41,100 DEBUG [PEWorker-1 {}] assignment.GCRegionProcedure(87): Archiving region=662b2b4ec89923845e6b3b779abe59af 2024-11-10T03:13:41,100 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T03:13:41,100 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T03:13:41,100 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 09efdeef9bad77e33c2f27b176a37faf, NAME => 'testScanDoesNotCleanRegionsWithExistingParents,aaa,1731208418043.09efdeef9bad77e33c2f27b176a37faf.', STARTKEY => 'aaa', ENDKEY => '', SPLIT => true}] 2024-11-10T03:13:41,100 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testScanDoesNotCleanRegionsWithExistingParents/data/default/testScanDoesNotCleanRegionsWithExistingParents/662b2b4ec89923845e6b3b779abe59af 2024-11-10T03:13:41,100 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=true; GCRegionProcedure table=testScanDoesNotCleanRegionsWithExistingParents, region=09efdeef9bad77e33c2f27b176a37faf in 3 msec 2024-11-10T03:13:41,101 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-10T03:13:41,101 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testScanDoesNotCleanRegionsWithExistingParents/data/default/testScanDoesNotCleanRegionsWithExistingParents/662b2b4ec89923845e6b3b779abe59af/cf] 2024-11-10T03:13:41,107 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.janitor.TestCatalogJanitor#testScanDoesNotCleanRegionsWithExistingParents Thread=35 (was 29) Potentially hanging thread: PEWorker-1 java.base@17.0.11/java.net.URI.quote(URI.java:2797) java.base@17.0.11/java.net.URI.appendSchemeSpecificPart(URI.java:2085) java.base@17.0.11/java.net.URI.toString(URI.java:2115) java.base@17.0.11/java.net.URI.(URI.java:777) app//org.apache.hadoop.fs.Path.initialize(Path.java:261) app//org.apache.hadoop.fs.Path.(Path.java:162) app//org.apache.hadoop.fs.Path.(Path.java:130) app//org.apache.hadoop.hbase.util.HFileArchiveUtil.getArchivePath(HFileArchiveUtil.java:185) app//org.apache.hadoop.hbase.util.HFileArchiveUtil.getTableArchivePath(HFileArchiveUtil.java:149) app//org.apache.hadoop.hbase.util.HFileArchiveUtil.getRegionArchiveDir(HFileArchiveUtil.java:133) app//org.apache.hadoop.hbase.backup.HFileArchiver.archiveRegion(HFileArchiver.java:135) app//org.apache.hadoop.hbase.backup.HFileArchiver.archiveRegion(HFileArchiver.java:100) app//org.apache.hadoop.hbase.master.assignment.GCRegionProcedure.executeFromState(GCRegionProcedure.java:89) app//org.apache.hadoop.hbase.master.assignment.GCRegionProcedure.executeFromState(GCRegionProcedure.java:51) app//org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) app//org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread$$Lambda$329/0x00007fe96455e6a8.call(Unknown Source) app//org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) Potentially hanging thread: AsyncFSWAL-0-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testScanDoesNotCleanRegionsWithExistingParents/MasterData-prefix:mockmaster.example.org,1234,-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSOutputFlusher-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testScanDoesNotCleanRegionsWithExistingParents/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731208417813 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PEWorker-2 app//org.apache.logging.log4j.core.appender.OutputStreamManager.writeBytes(OutputStreamManager.java:352) app//org.apache.logging.log4j.core.layout.TextEncoderHelper.writeEncodedText(TextEncoderHelper.java:96) app//org.apache.logging.log4j.core.layout.TextEncoderHelper.encodeText(TextEncoderHelper.java:65) app//org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:68) app//org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:32) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:227) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:59) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:215) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1521) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread$$Lambda$329/0x00007fe96455e6a8.call(Unknown Source) app//org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) - Thread LEAK? -, OpenFileDescriptor=399 (was 355) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=898 (was 967), ProcessCount=12 (was 11) - ProcessCount LEAK? -, AvailableMemoryMB=1956 (was 2086) 2024-11-10T03:13:41,112 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.janitor.TestCatalogJanitor#testDuplicateHFileResolution Thread=35, OpenFileDescriptor=397, MaxFileDescriptor=1048576, SystemLoadAverage=898, ProcessCount=12, AvailableMemoryMB=1953 2024-11-10T03:13:41,113 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testScanDoesNotCleanRegionsWithExistingParents/data/default/testScanDoesNotCleanRegionsWithExistingParents/662b2b4ec89923845e6b3b779abe59af 2024-11-10T03:13:41,124 INFO [Time-limited test {}] util.FSUtils(489): Created version file at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution with version=8 2024-11-10T03:13:41,125 DEBUG [Time-limited test {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/hbase.id] with ID: a956d5d2-24da-469f-96ee-02a28b23ff3b 2024-11-10T03:13:41,125 DEBUG [Time-limited test {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/.tmp/hbase.id 2024-11-10T03:13:41,130 DEBUG [Time-limited test {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/hbase.id] 2024-11-10T03:13:41,161 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T03:13:41,162 INFO [Time-limited test {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T03:13:41,165 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T03:13:41,166 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T03:13:41,191 INFO [Time-limited test {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/MasterData/data/master/store 2024-11-10T03:13:41,203 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:13:41,203 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T03:13:41,203 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:41,203 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:41,203 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T03:13:41,204 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:41,204 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:41,204 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731208421203Disabling compacts and flushes for region at 1731208421203Disabling writes for close at 1731208421204 (+1 ms)Writing region close event to WAL at 1731208421204Closed at 1731208421204 2024-11-10T03:13:41,204 WARN [Time-limited test {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/MasterData/data/master/store/.initializing 2024-11-10T03:13:41,204 DEBUG [Time-limited test {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/MasterData/WALs/mockmaster.example.org,1234,-1 2024-11-10T03:13:41,209 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T03:13:41,216 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=mockmaster.example.org%2C1234%2C-1, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/MasterData/WALs/mockmaster.example.org,1234,-1, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/MasterData/oldWALs, maxLogs=10 2024-11-10T03:13:41,235 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731208421216 2024-11-10T03:13:41,235 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T03:13:41,235 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T03:13:41,236 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:13:41,236 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:41,236 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:41,237 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:41,241 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T03:13:41,241 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:41,241 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T03:13:41,241 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:41,246 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T03:13:41,246 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:41,247 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:13:41,247 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:41,251 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T03:13:41,251 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:41,251 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:13:41,252 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:41,257 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T03:13:41,257 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:41,258 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:13:41,258 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:41,259 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:41,259 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:41,259 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:41,259 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:41,260 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T03:13:41,260 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:41,270 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T03:13:41,270 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60478186, jitterRate=-0.09880480170249939}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T03:13:41,270 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731208421237Initializing all the Stores at 1731208421237Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731208421237Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208421237Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208421237Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208421237Cleaning up temporary data from old regions at 1731208421259 (+22 ms)Region opened successfully at 1731208421270 (+11 ms) 2024-11-10T03:13:41,271 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T03:13:41,276 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T03:13:41,276 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered NoopProcedureStore lease in 0 msec 2024-11-10T03:13:41,276 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded NoopProcedureStore in 0 msec 2024-11-10T03:13:41,277 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731208451277 2024-11-10T03:13:41,277 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T03:13:41,279 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,100,1 2024-11-10T03:13:41,279 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,101,1 2024-11-10T03:13:41,279 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,102,1 2024-11-10T03:13:41,279 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,103,1 2024-11-10T03:13:41,280 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,104,1 2024-11-10T03:13:41,280 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,105,1 2024-11-10T03:13:41,280 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,106,1 2024-11-10T03:13:41,280 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,107,1 2024-11-10T03:13:41,280 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,108,1 2024-11-10T03:13:41,280 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,109,1 2024-11-10T03:13:41,281 INFO [Time-limited test {}] janitor.TestCatalogJanitor(639): Old root:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution 2024-11-10T03:13:41,281 INFO [Time-limited test {}] janitor.TestCatalogJanitor(640): Old table:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution 2024-11-10T03:13:41,282 INFO [Time-limited test {}] janitor.TestCatalogJanitor(641): Old store:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution/adef879cedd4e088c634959941e17d68/cf 2024-11-10T03:13:41,282 INFO [Time-limited test {}] janitor.TestCatalogJanitor(645): Old archive:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/archive/data/default/testDuplicateHFileResolution/adef879cedd4e088c634959941e17d68/cf 2024-11-10T03:13:41,313 DEBUG [Time-limited test {}] janitor.TestCatalogJanitor(711): Adding 2 store files to the storedir:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution/adef879cedd4e088c634959941e17d68/cf 2024-11-10T03:13:41,314 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => adef879cedd4e088c634959941e17d68, NAME => 'testDuplicateHFileResolution,aaa,1731208421281.adef879cedd4e088c634959941e17d68.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T03:13:41,315 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:41,316 DEBUG [Time-limited test {}] janitor.CatalogJanitor(351): Deleting region adef879cedd4e088c634959941e17d68 because daughters -- adef879cedd4e088c634959941e17d68, ddf7868028599f0f45779675a3204d9a -- no longer hold references 2024-11-10T03:13:41,316 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testDuplicateHFileResolution, region=adef879cedd4e088c634959941e17d68 2024-11-10T03:13:41,317 DEBUG [Time-limited test {}] janitor.CatalogJanitor(358): Submitted procedure pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testDuplicateHFileResolution, region=adef879cedd4e088c634959941e17d68 for split parent {ENCODED => adef879cedd4e088c634959941e17d68, NAME => 'testDuplicateHFileResolution,aaa,1731208421281.adef879cedd4e088c634959941e17d68.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T03:13:41,317 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testDuplicateHFileResolution, region=adef879cedd4e088c634959941e17d68 2024-11-10T03:13:41,317 DEBUG [PEWorker-1 {}] assignment.GCRegionProcedure(87): Archiving region=adef879cedd4e088c634959941e17d68 2024-11-10T03:13:41,318 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution/adef879cedd4e088c634959941e17d68 2024-11-10T03:13:41,318 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution/adef879cedd4e088c634959941e17d68/cf] 2024-11-10T03:13:41,349 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution/adef879cedd4e088c634959941e17d68/cf/_store0 to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/archive/data/default/testDuplicateHFileResolution/adef879cedd4e088c634959941e17d68/cf/_store0 2024-11-10T03:13:41,350 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution/adef879cedd4e088c634959941e17d68/cf/_store1 to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/archive/data/default/testDuplicateHFileResolution/adef879cedd4e088c634959941e17d68/cf/_store1 2024-11-10T03:13:41,351 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution/adef879cedd4e088c634959941e17d68 2024-11-10T03:13:41,353 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testDuplicateHFileResolution,aaa,1731208421281.adef879cedd4e088c634959941e17d68.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731208421352"}]},"ts":"9223372036854775807"} 2024-11-10T03:13:41,353 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T03:13:41,353 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => adef879cedd4e088c634959941e17d68, NAME => 'testDuplicateHFileResolution,aaa,1731208421281.adef879cedd4e088c634959941e17d68.', STARTKEY => 'aaa', ENDKEY => 'eee'}] 2024-11-10T03:13:41,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; GCRegionProcedure table=testDuplicateHFileResolution, region=adef879cedd4e088c634959941e17d68 in 37 msec 2024-11-10T03:13:41,605 DEBUG [Time-limited test {}] janitor.TestCatalogJanitor(711): Adding 2 store files to the storedir:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution/adef879cedd4e088c634959941e17d68/cf 2024-11-10T03:13:41,606 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => adef879cedd4e088c634959941e17d68, NAME => 'testDuplicateHFileResolution,aaa,1731208421281.adef879cedd4e088c634959941e17d68.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T03:13:41,607 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:41,608 DEBUG [Time-limited test {}] janitor.CatalogJanitor(351): Deleting region adef879cedd4e088c634959941e17d68 because daughters -- adef879cedd4e088c634959941e17d68, ddf7868028599f0f45779675a3204d9a -- no longer hold references 2024-11-10T03:13:41,609 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=2, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testDuplicateHFileResolution, region=adef879cedd4e088c634959941e17d68 2024-11-10T03:13:41,609 DEBUG [Time-limited test {}] janitor.CatalogJanitor(358): Submitted procedure pid=2, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testDuplicateHFileResolution, region=adef879cedd4e088c634959941e17d68 for split parent {ENCODED => adef879cedd4e088c634959941e17d68, NAME => 'testDuplicateHFileResolution,aaa,1731208421281.adef879cedd4e088c634959941e17d68.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T03:13:41,609 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testDuplicateHFileResolution, region=adef879cedd4e088c634959941e17d68 2024-11-10T03:13:41,610 DEBUG [PEWorker-2 {}] assignment.GCRegionProcedure(87): Archiving region=adef879cedd4e088c634959941e17d68 2024-11-10T03:13:41,610 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution/adef879cedd4e088c634959941e17d68 2024-11-10T03:13:41,611 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution/adef879cedd4e088c634959941e17d68/cf] 2024-11-10T03:13:41,612 ERROR [PEWorker-2 {}] backup.HFileArchiver(523): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/archive/data/default/testDuplicateHFileResolution/adef879cedd4e088c634959941e17d68/cf/_store0 already exists in archive, moving to timestamped backup and overwriting current FileablePath, file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution/adef879cedd4e088c634959941e17d68/cf/_store0. archiveLen: 12 currentLen: 12 archiveMtime: 1731208421349 currentMtime: 1731208421588 2024-11-10T03:13:41,613 INFO [PEWorker-2 {}] backup.HFileArchiver(540): Backed up archive file from file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/archive/data/default/testDuplicateHFileResolution/adef879cedd4e088c634959941e17d68/cf/_store0 to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/archive/data/default/testDuplicateHFileResolution/adef879cedd4e088c634959941e17d68/cf/_store0.1731208421611. 2024-11-10T03:13:41,614 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution/adef879cedd4e088c634959941e17d68/cf/_store0 to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/archive/data/default/testDuplicateHFileResolution/adef879cedd4e088c634959941e17d68/cf/_store0 2024-11-10T03:13:41,614 ERROR [PEWorker-2 {}] backup.HFileArchiver(523): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/archive/data/default/testDuplicateHFileResolution/adef879cedd4e088c634959941e17d68/cf/_store1 already exists in archive, moving to timestamped backup and overwriting current FileablePath, file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution/adef879cedd4e088c634959941e17d68/cf/_store1. archiveLen: 12 currentLen: 12 archiveMtime: 1731208421350 currentMtime: 1731208421600 2024-11-10T03:13:41,615 INFO [PEWorker-2 {}] backup.HFileArchiver(540): Backed up archive file from file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/archive/data/default/testDuplicateHFileResolution/adef879cedd4e088c634959941e17d68/cf/_store1 to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/archive/data/default/testDuplicateHFileResolution/adef879cedd4e088c634959941e17d68/cf/_store1.1731208421611. 2024-11-10T03:13:41,615 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution/adef879cedd4e088c634959941e17d68/cf/_store1 to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/archive/data/default/testDuplicateHFileResolution/adef879cedd4e088c634959941e17d68/cf/_store1 2024-11-10T03:13:41,616 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution/adef879cedd4e088c634959941e17d68 2024-11-10T03:13:41,617 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testDuplicateHFileResolution,aaa,1731208421281.adef879cedd4e088c634959941e17d68.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731208421616"}]},"ts":"9223372036854775807"} 2024-11-10T03:13:41,618 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T03:13:41,618 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => adef879cedd4e088c634959941e17d68, NAME => 'testDuplicateHFileResolution,aaa,1731208421281.adef879cedd4e088c634959941e17d68.', STARTKEY => 'aaa', ENDKEY => 'eee'}] 2024-11-10T03:13:41,618 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, state=SUCCESS, hasLock=false; GCRegionProcedure table=testDuplicateHFileResolution, region=adef879cedd4e088c634959941e17d68 in 9 msec 2024-11-10T03:13:41,862 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T03:13:41,862 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T03:13:41,863 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T03:13:41,873 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.janitor.TestCatalogJanitor#testDuplicateHFileResolution Thread=38 (was 35) Potentially hanging thread: AsyncFSWAL-0-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/MasterData-prefix:mockmaster.example.org,1234,-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSOutputFlusher-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDuplicateHFileResolution/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731208421216 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=423 (was 397) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=898 (was 898), ProcessCount=11 (was 12), AvailableMemoryMB=1868 (was 1953) 2024-11-10T03:13:41,884 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.janitor.TestCatalogJanitor#testAlreadyRunningStatus Thread=38, OpenFileDescriptor=423, MaxFileDescriptor=1048576, SystemLoadAverage=898, ProcessCount=11, AvailableMemoryMB=1864 2024-11-10T03:13:41,908 INFO [Time-limited test {}] util.FSUtils(489): Created version file at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testAlreadyRunningStatus with version=8 2024-11-10T03:13:41,909 DEBUG [Time-limited test {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testAlreadyRunningStatus/hbase.id] with ID: 39ad7a7f-b23f-4a60-b1d0-fefb47049f7e 2024-11-10T03:13:41,909 DEBUG [Time-limited test {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testAlreadyRunningStatus/.tmp/hbase.id 2024-11-10T03:13:41,929 DEBUG [Time-limited test {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testAlreadyRunningStatus/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testAlreadyRunningStatus/hbase.id] 2024-11-10T03:13:42,020 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T03:13:42,021 INFO [Time-limited test {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T03:13:42,023 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T03:13:42,024 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T03:13:42,085 INFO [Time-limited test {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testAlreadyRunningStatus/MasterData/data/master/store 2024-11-10T03:13:42,121 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:13:42,121 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T03:13:42,121 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:42,121 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:42,122 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T03:13:42,122 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:42,122 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:42,122 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731208422121Disabling compacts and flushes for region at 1731208422121Disabling writes for close at 1731208422122 (+1 ms)Writing region close event to WAL at 1731208422122Closed at 1731208422122 2024-11-10T03:13:42,124 WARN [Time-limited test {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testAlreadyRunningStatus/MasterData/data/master/store/.initializing 2024-11-10T03:13:42,125 DEBUG [Time-limited test {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testAlreadyRunningStatus/MasterData/WALs/mockmaster.example.org,1234,-1 2024-11-10T03:13:42,141 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T03:13:42,148 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=mockmaster.example.org%2C1234%2C-1, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testAlreadyRunningStatus/MasterData/WALs/mockmaster.example.org,1234,-1, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testAlreadyRunningStatus/MasterData/oldWALs, maxLogs=10 2024-11-10T03:13:42,179 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testAlreadyRunningStatus/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731208422148 2024-11-10T03:13:42,180 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T03:13:42,180 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T03:13:42,180 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:13:42,181 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:42,181 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:42,182 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:42,191 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T03:13:42,192 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:42,192 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T03:13:42,192 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:42,197 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T03:13:42,197 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:42,198 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:13:42,198 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:42,205 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T03:13:42,205 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:42,206 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:13:42,206 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:42,210 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T03:13:42,210 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:42,210 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:13:42,210 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:42,211 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testAlreadyRunningStatus/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:42,212 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testAlreadyRunningStatus/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:42,212 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:42,212 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:42,213 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T03:13:42,213 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:42,238 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testAlreadyRunningStatus/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T03:13:42,238 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63173644, jitterRate=-0.058639347553253174}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T03:13:42,238 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731208422181Initializing all the Stores at 1731208422182 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731208422182Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208422182Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208422182Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208422182Cleaning up temporary data from old regions at 1731208422212 (+30 ms)Region opened successfully at 1731208422238 (+26 ms) 2024-11-10T03:13:42,239 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T03:13:42,244 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T03:13:42,245 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered NoopProcedureStore lease in 0 msec 2024-11-10T03:13:42,245 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded NoopProcedureStore in 0 msec 2024-11-10T03:13:42,251 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731208452251 2024-11-10T03:13:42,251 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T03:13:42,257 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,100,1 2024-11-10T03:13:42,257 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,101,1 2024-11-10T03:13:42,257 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,102,1 2024-11-10T03:13:42,257 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,103,1 2024-11-10T03:13:42,258 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,104,1 2024-11-10T03:13:42,258 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,105,1 2024-11-10T03:13:42,258 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,106,1 2024-11-10T03:13:42,258 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,107,1 2024-11-10T03:13:42,258 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,108,1 2024-11-10T03:13:42,259 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,109,1 2024-11-10T03:13:42,261 DEBUG [Thread-266 {}] janitor.CatalogJanitor(170): CatalogJanitor already running 2024-11-10T03:13:42,264 DEBUG [Thread-265 {}] janitor.CatalogJanitor(180): 2024-11-10T03:13:42,264 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T03:13:42,265 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T03:13:42,266 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T03:13:42,275 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.janitor.TestCatalogJanitor#testAlreadyRunningStatus Thread=42 (was 38) Potentially hanging thread: AsyncFSOutputFlusher-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testAlreadyRunningStatus/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731208422148 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testAlreadyRunningStatus/MasterData-prefix:mockmaster.example.org,1234,-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 423) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=898 (was 898), ProcessCount=11 (was 11), AvailableMemoryMB=1688 (was 1864) 2024-11-10T03:13:42,282 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.janitor.TestCatalogJanitor#testParentCleanedEvenIfDaughterGoneFirst Thread=42, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=898, ProcessCount=11, AvailableMemoryMB=1688 2024-11-10T03:13:42,301 INFO [Time-limited test {}] util.FSUtils(489): Created version file at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testParentCleanedEvenIfDaughterGoneFirst with version=8 2024-11-10T03:13:42,302 DEBUG [Time-limited test {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testParentCleanedEvenIfDaughterGoneFirst/hbase.id] with ID: a9970a3a-ccca-4347-b4c6-15b1f9c5f3d1 2024-11-10T03:13:42,302 DEBUG [Time-limited test {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testParentCleanedEvenIfDaughterGoneFirst/.tmp/hbase.id 2024-11-10T03:13:42,309 DEBUG [Time-limited test {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testParentCleanedEvenIfDaughterGoneFirst/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testParentCleanedEvenIfDaughterGoneFirst/hbase.id] 2024-11-10T03:13:42,378 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T03:13:42,380 INFO [Time-limited test {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T03:13:42,383 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T03:13:42,384 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T03:13:42,426 INFO [Time-limited test {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testParentCleanedEvenIfDaughterGoneFirst/MasterData/data/master/store 2024-11-10T03:13:42,440 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:13:42,441 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T03:13:42,441 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:42,441 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:42,441 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T03:13:42,441 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:42,442 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:42,442 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731208422441Disabling compacts and flushes for region at 1731208422441Disabling writes for close at 1731208422441Writing region close event to WAL at 1731208422441Closed at 1731208422441 2024-11-10T03:13:42,442 WARN [Time-limited test {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testParentCleanedEvenIfDaughterGoneFirst/MasterData/data/master/store/.initializing 2024-11-10T03:13:42,442 DEBUG [Time-limited test {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testParentCleanedEvenIfDaughterGoneFirst/MasterData/WALs/mockmaster.example.org,1234,-1 2024-11-10T03:13:42,457 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T03:13:42,468 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=mockmaster.example.org%2C1234%2C-1, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testParentCleanedEvenIfDaughterGoneFirst/MasterData/WALs/mockmaster.example.org,1234,-1, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testParentCleanedEvenIfDaughterGoneFirst/MasterData/oldWALs, maxLogs=10 2024-11-10T03:13:42,501 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testParentCleanedEvenIfDaughterGoneFirst/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731208422468 2024-11-10T03:13:42,502 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T03:13:42,502 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T03:13:42,503 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:13:42,503 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:42,503 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:42,504 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:42,516 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T03:13:42,517 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:42,517 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T03:13:42,518 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:42,525 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T03:13:42,525 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:42,526 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:13:42,526 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:42,533 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T03:13:42,533 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:42,534 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:13:42,534 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:42,544 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T03:13:42,544 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:42,544 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:13:42,545 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:42,546 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testParentCleanedEvenIfDaughterGoneFirst/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:42,546 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testParentCleanedEvenIfDaughterGoneFirst/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:42,546 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:42,547 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:42,547 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T03:13:42,548 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:42,566 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testParentCleanedEvenIfDaughterGoneFirst/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T03:13:42,567 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72580994, jitterRate=0.08154109120368958}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T03:13:42,568 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731208422503Initializing all the Stores at 1731208422504 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731208422504Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208422504Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208422504Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208422504Cleaning up temporary data from old regions at 1731208422547 (+43 ms)Region opened successfully at 1731208422568 (+21 ms) 2024-11-10T03:13:42,568 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T03:13:42,576 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T03:13:42,577 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered NoopProcedureStore lease in 0 msec 2024-11-10T03:13:42,577 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded NoopProcedureStore in 0 msec 2024-11-10T03:13:42,623 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731208452623 2024-11-10T03:13:42,624 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T03:13:42,627 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,100,1 2024-11-10T03:13:42,628 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,101,1 2024-11-10T03:13:42,628 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,102,1 2024-11-10T03:13:42,628 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,103,1 2024-11-10T03:13:42,628 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,104,1 2024-11-10T03:13:42,628 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,105,1 2024-11-10T03:13:42,628 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,106,1 2024-11-10T03:13:42,628 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,107,1 2024-11-10T03:13:42,628 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,108,1 2024-11-10T03:13:42,629 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,109,1 2024-11-10T03:13:45,431 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-10T03:13:45,431 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-10T03:13:45,634 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:45,654 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => 54187daad718881250de0b6c94c78c3f, NAME => 'testParentCleanedEvenIfDaughterGoneFirst,aaa,1731208422629.54187daad718881250de0b6c94c78c3f.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T03:13:45,656 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:45,656 DEBUG [Time-limited test {}] janitor.CatalogJanitor(364): Deferring removal of region {ENCODED => 54187daad718881250de0b6c94c78c3f, NAME => 'testParentCleanedEvenIfDaughterGoneFirst,aaa,1731208422629.54187daad718881250de0b6c94c78c3f.', STARTKEY => 'aaa', ENDKEY => 'eee'} because daughter {ENCODED => 993e310cda38896bbecfad1fc520982f, NAME => 'testParentCleanedEvenIfDaughterGoneFirst,aaa,1731208423630.993e310cda38896bbecfad1fc520982f.', STARTKEY => 'aaa', ENDKEY => 'ccc'} still has references 2024-11-10T03:13:45,656 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => 80d072f3a49ef5ae03833efa089fcedc, NAME => 'testParentCleanedEvenIfDaughterGoneFirst,ccc,1731208424631.80d072f3a49ef5ae03833efa089fcedc.', STARTKEY => 'ccc', ENDKEY => 'eee'} 2024-11-10T03:13:45,657 DEBUG [Time-limited test {}] janitor.CatalogJanitor(351): Deleting region 80d072f3a49ef5ae03833efa089fcedc because daughters -- 27045286b0adc51b76c72c67b19ea17c, 1b15f24303aca3f7dca1e0f87d59d95a -- no longer hold references 2024-11-10T03:13:45,657 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testParentCleanedEvenIfDaughterGoneFirst, region=80d072f3a49ef5ae03833efa089fcedc 2024-11-10T03:13:45,658 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testParentCleanedEvenIfDaughterGoneFirst, region=80d072f3a49ef5ae03833efa089fcedc 2024-11-10T03:13:45,659 DEBUG [Time-limited test {}] janitor.CatalogJanitor(358): Submitted procedure pid=1, state=RUNNABLE:GC_REGION_PURGE_METADATA, hasLock=true; GCRegionProcedure table=testParentCleanedEvenIfDaughterGoneFirst, region=80d072f3a49ef5ae03833efa089fcedc for split parent {ENCODED => 80d072f3a49ef5ae03833efa089fcedc, NAME => 'testParentCleanedEvenIfDaughterGoneFirst,ccc,1731208424631.80d072f3a49ef5ae03833efa089fcedc.', STARTKEY => 'ccc', ENDKEY => 'eee'} 2024-11-10T03:13:45,659 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testParentCleanedEvenIfDaughterGoneFirst,ccc,1731208424631.80d072f3a49ef5ae03833efa089fcedc.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731208425658"}]},"ts":"9223372036854775807"} 2024-11-10T03:13:45,659 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T03:13:45,660 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 80d072f3a49ef5ae03833efa089fcedc, NAME => 'testParentCleanedEvenIfDaughterGoneFirst,ccc,1731208424631.80d072f3a49ef5ae03833efa089fcedc.', STARTKEY => 'ccc', ENDKEY => 'eee'}] 2024-11-10T03:13:45,660 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:45,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; GCRegionProcedure table=testParentCleanedEvenIfDaughterGoneFirst, region=80d072f3a49ef5ae03833efa089fcedc in 3 msec 2024-11-10T03:13:45,674 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:45,698 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => 993e310cda38896bbecfad1fc520982f, NAME => 'testParentCleanedEvenIfDaughterGoneFirst,aaa,1731208423630.993e310cda38896bbecfad1fc520982f.', STARTKEY => 'aaa', ENDKEY => 'ccc'} 2024-11-10T03:13:45,700 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:45,701 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:45,701 DEBUG [Time-limited test {}] janitor.CatalogJanitor(364): Deferring removal of region {ENCODED => 993e310cda38896bbecfad1fc520982f, NAME => 'testParentCleanedEvenIfDaughterGoneFirst,aaa,1731208423630.993e310cda38896bbecfad1fc520982f.', STARTKEY => 'aaa', ENDKEY => 'ccc'} because daughter {ENCODED => bd207b01d588e902058749d29a95570e, NAME => 'testParentCleanedEvenIfDaughterGoneFirst,aaa,1731208424631.bd207b01d588e902058749d29a95570e.', STARTKEY => 'aaa', ENDKEY => 'bbb'} still has references 2024-11-10T03:13:45,701 DEBUG [Time-limited test {}] janitor.CatalogJanitor(368): Deferring removal of region {ENCODED => 993e310cda38896bbecfad1fc520982f, NAME => 'testParentCleanedEvenIfDaughterGoneFirst,aaa,1731208423630.993e310cda38896bbecfad1fc520982f.', STARTKEY => 'aaa', ENDKEY => 'ccc'} because daughter {ENCODED => 0dbd48a910519e2ae24cace8f37c730d, NAME => 'testParentCleanedEvenIfDaughterGoneFirst,bbb,1731208424631.0dbd48a910519e2ae24cace8f37c730d.', STARTKEY => 'bbb', ENDKEY => 'ccc'} still has references 2024-11-10T03:13:45,702 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => 993e310cda38896bbecfad1fc520982f, NAME => 'testParentCleanedEvenIfDaughterGoneFirst,aaa,1731208423630.993e310cda38896bbecfad1fc520982f.', STARTKEY => 'aaa', ENDKEY => 'ccc'} 2024-11-10T03:13:45,703 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:45,704 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:45,704 DEBUG [Time-limited test {}] janitor.CatalogJanitor(351): Deleting region 993e310cda38896bbecfad1fc520982f because daughters -- bd207b01d588e902058749d29a95570e, 0dbd48a910519e2ae24cace8f37c730d -- no longer hold references 2024-11-10T03:13:45,704 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=2, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testParentCleanedEvenIfDaughterGoneFirst, region=993e310cda38896bbecfad1fc520982f 2024-11-10T03:13:45,704 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testParentCleanedEvenIfDaughterGoneFirst, region=993e310cda38896bbecfad1fc520982f 2024-11-10T03:13:45,705 DEBUG [PEWorker-2 {}] assignment.GCRegionProcedure(87): Archiving region=993e310cda38896bbecfad1fc520982f 2024-11-10T03:13:45,705 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testParentCleanedEvenIfDaughterGoneFirst/data/default/testParentCleanedEvenIfDaughterGoneFirst/993e310cda38896bbecfad1fc520982f 2024-11-10T03:13:45,706 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testParentCleanedEvenIfDaughterGoneFirst/data/default/testParentCleanedEvenIfDaughterGoneFirst/993e310cda38896bbecfad1fc520982f/cf] 2024-11-10T03:13:45,706 DEBUG [Time-limited test {}] janitor.CatalogJanitor(358): Submitted procedure pid=2, state=RUNNABLE:GC_REGION_ARCHIVE, hasLock=true; GCRegionProcedure table=testParentCleanedEvenIfDaughterGoneFirst, region=993e310cda38896bbecfad1fc520982f for split parent {ENCODED => 993e310cda38896bbecfad1fc520982f, NAME => 'testParentCleanedEvenIfDaughterGoneFirst,aaa,1731208423630.993e310cda38896bbecfad1fc520982f.', STARTKEY => 'aaa', ENDKEY => 'ccc'} 2024-11-10T03:13:45,706 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => 54187daad718881250de0b6c94c78c3f, NAME => 'testParentCleanedEvenIfDaughterGoneFirst,aaa,1731208422629.54187daad718881250de0b6c94c78c3f.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T03:13:45,707 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:45,708 DEBUG [Time-limited test {}] janitor.CatalogJanitor(351): Deleting region 54187daad718881250de0b6c94c78c3f because daughters -- 993e310cda38896bbecfad1fc520982f, 80d072f3a49ef5ae03833efa089fcedc -- no longer hold references 2024-11-10T03:13:45,708 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=3, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testParentCleanedEvenIfDaughterGoneFirst, region=54187daad718881250de0b6c94c78c3f 2024-11-10T03:13:45,708 DEBUG [Time-limited test {}] janitor.CatalogJanitor(358): Submitted procedure pid=3, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testParentCleanedEvenIfDaughterGoneFirst, region=54187daad718881250de0b6c94c78c3f for split parent {ENCODED => 54187daad718881250de0b6c94c78c3f, NAME => 'testParentCleanedEvenIfDaughterGoneFirst,aaa,1731208422629.54187daad718881250de0b6c94c78c3f.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T03:13:45,708 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T03:13:45,708 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=3, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testParentCleanedEvenIfDaughterGoneFirst, region=54187daad718881250de0b6c94c78c3f 2024-11-10T03:13:45,708 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T03:13:45,708 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T03:13:45,721 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.janitor.TestCatalogJanitor#testParentCleanedEvenIfDaughterGoneFirst Thread=52 (was 42) Potentially hanging thread: process reaper (pid 2685) java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testParentCleanedEvenIfDaughterGoneFirst/MasterData-prefix:mockmaster.example.org,1234,-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PEWorker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167) app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2176) Potentially hanging thread: PEWorker-2 java.base@17.0.11/java.lang.ProcessImpl.forkAndExec(Native Method) java.base@17.0.11/java.lang.ProcessImpl.(ProcessImpl.java:314) java.base@17.0.11/java.lang.ProcessImpl.start(ProcessImpl.java:244) java.base@17.0.11/java.lang.ProcessBuilder.start(ProcessBuilder.java:1110) java.base@17.0.11/java.lang.ProcessBuilder.start(ProcessBuilder.java:1073) app//org.apache.hadoop.util.Shell.runCommand(Shell.java:998) app//org.apache.hadoop.util.Shell.run(Shell.java:959) app//org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) app//org.apache.hadoop.util.Shell.execCommand(Shell.java:1377) app//org.apache.hadoop.util.Shell.execCommand(Shell.java:1359) app//org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:1116) app//org.apache.hadoop.fs.RawLocalFileSystem.mkOneDirWithMode(RawLocalFileSystem.java:798) app//org.apache.hadoop.fs.RawLocalFileSystem.mkdirsWithOptionalPermission(RawLocalFileSystem.java:838) app//org.apache.hadoop.fs.RawLocalFileSystem.mkdirs(RawLocalFileSystem.java:810) app//org.apache.hadoop.fs.RawLocalFileSystem.mkdirsWithOptionalPermission(RawLocalFileSystem.java:837) app//org.apache.hadoop.fs.RawLocalFileSystem.mkdirs(RawLocalFileSystem.java:810) app//org.apache.hadoop.fs.RawLocalFileSystem.mkdirsWithOptionalPermission(RawLocalFileSystem.java:837) app//org.apache.hadoop.fs.RawLocalFileSystem.mkdirs(RawLocalFileSystem.java:810) app//org.apache.hadoop.fs.RawLocalFileSystem.mkdirsWithOptionalPermission(RawLocalFileSystem.java:837) app//org.apache.hadoop.fs.RawLocalFileSystem.mkdirs(RawLocalFileSystem.java:810) app//org.apache.hadoop.fs.ChecksumFileSystem.mkdirs(ChecksumFileSystem.java:988) app//org.apache.hadoop.hbase.backup.HFileArchiver.resolveAndArchive(HFileArchiver.java:433) app//org.apache.hadoop.hbase.backup.HFileArchiver.archiveRegion(HFileArchiver.java:161) app//org.apache.hadoop.hbase.backup.HFileArchiver.archiveRegion(HFileArchiver.java:100) app//org.apache.hadoop.hbase.master.assignment.GCRegionProcedure.executeFromState(GCRegionProcedure.java:89) app//org.apache.hadoop.hbase.master.assignment.GCRegionProcedure.executeFromState(GCRegionProcedure.java:51) app//org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) app//org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread$$Lambda$329/0x00007fe96455e6a8.call(Unknown Source) app//org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) Potentially hanging thread: AsyncFSOutputFlusher-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testParentCleanedEvenIfDaughterGoneFirst/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731208422468 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureDispatcherTimeoutThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.procedure2.util.DelayedUtil.takeWithoutInterrupt(DelayedUtil.java:81) app//org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher$TimeoutExecutorThread.run(RemoteProcedureDispatcher.java:328) Potentially hanging thread: ProcExecTimeout java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:274) app//org.apache.hadoop.hbase.procedure2.util.DelayedUtil.takeWithoutInterrupt(DelayedUtil.java:81) app//org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.run(TimeoutExecutorThread.java:56) Potentially hanging thread: PEWorker-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167) app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2176) - Thread LEAK? -, OpenFileDescriptor=494 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=866 (was 898), ProcessCount=11 (was 11), AvailableMemoryMB=1526 (was 1688) 2024-11-10T03:13:45,732 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.janitor.TestCatalogJanitor#testLastParentCleanedEvenIfDaughterGoneFirst Thread=48, OpenFileDescriptor=501, MaxFileDescriptor=1048576, SystemLoadAverage=866, ProcessCount=12, AvailableMemoryMB=1525 2024-11-10T03:13:45,773 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testParentCleanedEvenIfDaughterGoneFirst/data/default/testParentCleanedEvenIfDaughterGoneFirst/993e310cda38896bbecfad1fc520982f 2024-11-10T03:13:45,821 INFO [Time-limited test {}] util.FSUtils(489): Created version file at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testLastParentCleanedEvenIfDaughterGoneFirst with version=8 2024-11-10T03:13:45,821 DEBUG [Time-limited test {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testLastParentCleanedEvenIfDaughterGoneFirst/hbase.id] with ID: 6645e222-0973-4ded-a1cb-201210784fb6 2024-11-10T03:13:45,821 DEBUG [Time-limited test {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testLastParentCleanedEvenIfDaughterGoneFirst/.tmp/hbase.id 2024-11-10T03:13:45,845 DEBUG [Time-limited test {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testLastParentCleanedEvenIfDaughterGoneFirst/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testLastParentCleanedEvenIfDaughterGoneFirst/hbase.id] 2024-11-10T03:13:45,904 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T03:13:45,904 INFO [Time-limited test {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T03:13:45,906 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T03:13:45,907 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T03:13:45,973 INFO [Time-limited test {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testLastParentCleanedEvenIfDaughterGoneFirst/MasterData/data/master/store 2024-11-10T03:13:45,983 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:13:45,983 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T03:13:45,983 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:45,983 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:45,983 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T03:13:45,983 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:45,983 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:45,983 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731208425983Disabling compacts and flushes for region at 1731208425983Disabling writes for close at 1731208425983Writing region close event to WAL at 1731208425983Closed at 1731208425983 2024-11-10T03:13:45,983 WARN [Time-limited test {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testLastParentCleanedEvenIfDaughterGoneFirst/MasterData/data/master/store/.initializing 2024-11-10T03:13:45,984 DEBUG [Time-limited test {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testLastParentCleanedEvenIfDaughterGoneFirst/MasterData/WALs/mockmaster.example.org,1234,-1 2024-11-10T03:13:45,989 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T03:13:45,992 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=mockmaster.example.org%2C1234%2C-1, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testLastParentCleanedEvenIfDaughterGoneFirst/MasterData/WALs/mockmaster.example.org,1234,-1, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testLastParentCleanedEvenIfDaughterGoneFirst/MasterData/oldWALs, maxLogs=10 2024-11-10T03:13:46,016 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testLastParentCleanedEvenIfDaughterGoneFirst/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731208425993 2024-11-10T03:13:46,016 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T03:13:46,017 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T03:13:46,017 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:13:46,017 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:46,017 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:46,020 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:46,040 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T03:13:46,041 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:46,041 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T03:13:46,041 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:46,049 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T03:13:46,049 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:46,050 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:13:46,050 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:46,061 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T03:13:46,061 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:46,061 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:13:46,062 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:46,066 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T03:13:46,066 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:46,067 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:13:46,067 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:46,068 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testLastParentCleanedEvenIfDaughterGoneFirst/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:46,068 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testLastParentCleanedEvenIfDaughterGoneFirst/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:46,068 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:46,068 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:46,069 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T03:13:46,070 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:46,081 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testLastParentCleanedEvenIfDaughterGoneFirst/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T03:13:46,082 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60954950, jitterRate=-0.0917004644870758}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T03:13:46,082 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731208426018Initializing all the Stores at 1731208426018Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731208426018Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208426020 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208426020Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208426020Cleaning up temporary data from old regions at 1731208426069 (+49 ms)Region opened successfully at 1731208426082 (+13 ms) 2024-11-10T03:13:46,083 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T03:13:46,090 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T03:13:46,090 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered NoopProcedureStore lease in 0 msec 2024-11-10T03:13:46,091 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded NoopProcedureStore in 0 msec 2024-11-10T03:13:46,108 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731208456108 2024-11-10T03:13:46,108 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T03:13:46,121 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,100,1 2024-11-10T03:13:46,121 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,101,1 2024-11-10T03:13:46,121 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,102,1 2024-11-10T03:13:46,122 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,103,1 2024-11-10T03:13:46,122 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,104,1 2024-11-10T03:13:46,122 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,105,1 2024-11-10T03:13:46,122 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,106,1 2024-11-10T03:13:46,122 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,107,1 2024-11-10T03:13:46,122 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,108,1 2024-11-10T03:13:46,123 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,109,1 2024-11-10T03:13:49,128 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:49,180 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => f3c22b4565e27ca901871c0271500cac, NAME => 'testLastParentCleanedEvenIfDaughterGoneFirst,aaa,1731208426123.f3c22b4565e27ca901871c0271500cac.', STARTKEY => 'aaa', ENDKEY => ''} 2024-11-10T03:13:49,181 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:49,182 DEBUG [Time-limited test {}] janitor.CatalogJanitor(364): Deferring removal of region {ENCODED => f3c22b4565e27ca901871c0271500cac, NAME => 'testLastParentCleanedEvenIfDaughterGoneFirst,aaa,1731208426123.f3c22b4565e27ca901871c0271500cac.', STARTKEY => 'aaa', ENDKEY => ''} because daughter {ENCODED => 7d633c6c1926c131562f497764056809, NAME => 'testLastParentCleanedEvenIfDaughterGoneFirst,aaa,1731208427124.7d633c6c1926c131562f497764056809.', STARTKEY => 'aaa', ENDKEY => 'ccc'} still has references 2024-11-10T03:13:49,182 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => 3ebda5aadbec663b47b50f4ab4cac2b6, NAME => 'testLastParentCleanedEvenIfDaughterGoneFirst,ccc,1731208428125.3ebda5aadbec663b47b50f4ab4cac2b6.', STARTKEY => 'ccc', ENDKEY => ''} 2024-11-10T03:13:49,182 DEBUG [Time-limited test {}] janitor.CatalogJanitor(351): Deleting region 3ebda5aadbec663b47b50f4ab4cac2b6 because daughters -- 0508d41018f20998251c388b202de7c1, 26bb93247af5542979bca3969b9786e8 -- no longer hold references 2024-11-10T03:13:49,183 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testLastParentCleanedEvenIfDaughterGoneFirst, region=3ebda5aadbec663b47b50f4ab4cac2b6 2024-11-10T03:13:49,183 DEBUG [Time-limited test {}] janitor.CatalogJanitor(358): Submitted procedure pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testLastParentCleanedEvenIfDaughterGoneFirst, region=3ebda5aadbec663b47b50f4ab4cac2b6 for split parent {ENCODED => 3ebda5aadbec663b47b50f4ab4cac2b6, NAME => 'testLastParentCleanedEvenIfDaughterGoneFirst,ccc,1731208428125.3ebda5aadbec663b47b50f4ab4cac2b6.', STARTKEY => 'ccc', ENDKEY => ''} 2024-11-10T03:13:49,183 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testLastParentCleanedEvenIfDaughterGoneFirst, region=3ebda5aadbec663b47b50f4ab4cac2b6 2024-11-10T03:13:49,184 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:49,184 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testLastParentCleanedEvenIfDaughterGoneFirst,ccc,1731208428125.3ebda5aadbec663b47b50f4ab4cac2b6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731208429183"}]},"ts":"9223372036854775807"} 2024-11-10T03:13:49,185 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T03:13:49,185 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3ebda5aadbec663b47b50f4ab4cac2b6, NAME => 'testLastParentCleanedEvenIfDaughterGoneFirst,ccc,1731208428125.3ebda5aadbec663b47b50f4ab4cac2b6.', STARTKEY => 'ccc', ENDKEY => ''}] 2024-11-10T03:13:49,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; GCRegionProcedure table=testLastParentCleanedEvenIfDaughterGoneFirst, region=3ebda5aadbec663b47b50f4ab4cac2b6 in 3 msec 2024-11-10T03:13:49,208 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:49,223 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => 7d633c6c1926c131562f497764056809, NAME => 'testLastParentCleanedEvenIfDaughterGoneFirst,aaa,1731208427124.7d633c6c1926c131562f497764056809.', STARTKEY => 'aaa', ENDKEY => 'ccc'} 2024-11-10T03:13:49,224 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:49,225 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:49,226 DEBUG [Time-limited test {}] janitor.CatalogJanitor(364): Deferring removal of region {ENCODED => 7d633c6c1926c131562f497764056809, NAME => 'testLastParentCleanedEvenIfDaughterGoneFirst,aaa,1731208427124.7d633c6c1926c131562f497764056809.', STARTKEY => 'aaa', ENDKEY => 'ccc'} because daughter {ENCODED => e2bf5113fa409780489f7afcdcf4adf7, NAME => 'testLastParentCleanedEvenIfDaughterGoneFirst,aaa,1731208428125.e2bf5113fa409780489f7afcdcf4adf7.', STARTKEY => 'aaa', ENDKEY => 'bbb'} still has references 2024-11-10T03:13:49,226 DEBUG [Time-limited test {}] janitor.CatalogJanitor(368): Deferring removal of region {ENCODED => 7d633c6c1926c131562f497764056809, NAME => 'testLastParentCleanedEvenIfDaughterGoneFirst,aaa,1731208427124.7d633c6c1926c131562f497764056809.', STARTKEY => 'aaa', ENDKEY => 'ccc'} because daughter {ENCODED => 0b995f008921e8d9657a912108d4f455, NAME => 'testLastParentCleanedEvenIfDaughterGoneFirst,bbb,1731208428125.0b995f008921e8d9657a912108d4f455.', STARTKEY => 'bbb', ENDKEY => 'ccc'} still has references 2024-11-10T03:13:49,226 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => 7d633c6c1926c131562f497764056809, NAME => 'testLastParentCleanedEvenIfDaughterGoneFirst,aaa,1731208427124.7d633c6c1926c131562f497764056809.', STARTKEY => 'aaa', ENDKEY => 'ccc'} 2024-11-10T03:13:49,227 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:49,228 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:49,228 DEBUG [Time-limited test {}] janitor.CatalogJanitor(351): Deleting region 7d633c6c1926c131562f497764056809 because daughters -- e2bf5113fa409780489f7afcdcf4adf7, 0b995f008921e8d9657a912108d4f455 -- no longer hold references 2024-11-10T03:13:49,228 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=2, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testLastParentCleanedEvenIfDaughterGoneFirst, region=7d633c6c1926c131562f497764056809 2024-11-10T03:13:49,228 DEBUG [Time-limited test {}] janitor.CatalogJanitor(358): Submitted procedure pid=2, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testLastParentCleanedEvenIfDaughterGoneFirst, region=7d633c6c1926c131562f497764056809 for split parent {ENCODED => 7d633c6c1926c131562f497764056809, NAME => 'testLastParentCleanedEvenIfDaughterGoneFirst,aaa,1731208427124.7d633c6c1926c131562f497764056809.', STARTKEY => 'aaa', ENDKEY => 'ccc'} 2024-11-10T03:13:49,228 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => f3c22b4565e27ca901871c0271500cac, NAME => 'testLastParentCleanedEvenIfDaughterGoneFirst,aaa,1731208426123.f3c22b4565e27ca901871c0271500cac.', STARTKEY => 'aaa', ENDKEY => ''} 2024-11-10T03:13:49,228 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testLastParentCleanedEvenIfDaughterGoneFirst, region=7d633c6c1926c131562f497764056809 2024-11-10T03:13:49,229 DEBUG [PEWorker-2 {}] assignment.GCRegionProcedure(87): Archiving region=7d633c6c1926c131562f497764056809 2024-11-10T03:13:49,229 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testLastParentCleanedEvenIfDaughterGoneFirst/data/default/testLastParentCleanedEvenIfDaughterGoneFirst/7d633c6c1926c131562f497764056809 2024-11-10T03:13:49,229 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:49,229 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testLastParentCleanedEvenIfDaughterGoneFirst/data/default/testLastParentCleanedEvenIfDaughterGoneFirst/7d633c6c1926c131562f497764056809/cf] 2024-11-10T03:13:49,229 DEBUG [Time-limited test {}] janitor.CatalogJanitor(351): Deleting region f3c22b4565e27ca901871c0271500cac because daughters -- 7d633c6c1926c131562f497764056809, 3ebda5aadbec663b47b50f4ab4cac2b6 -- no longer hold references 2024-11-10T03:13:49,230 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=3, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testLastParentCleanedEvenIfDaughterGoneFirst, region=f3c22b4565e27ca901871c0271500cac 2024-11-10T03:13:49,230 DEBUG [Time-limited test {}] janitor.CatalogJanitor(358): Submitted procedure pid=3, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testLastParentCleanedEvenIfDaughterGoneFirst, region=f3c22b4565e27ca901871c0271500cac for split parent {ENCODED => f3c22b4565e27ca901871c0271500cac, NAME => 'testLastParentCleanedEvenIfDaughterGoneFirst,aaa,1731208426123.f3c22b4565e27ca901871c0271500cac.', STARTKEY => 'aaa', ENDKEY => ''} 2024-11-10T03:13:49,230 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T03:13:49,230 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T03:13:49,230 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=3, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testLastParentCleanedEvenIfDaughterGoneFirst, region=f3c22b4565e27ca901871c0271500cac 2024-11-10T03:13:49,230 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T03:13:49,237 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.janitor.TestCatalogJanitor#testLastParentCleanedEvenIfDaughterGoneFirst Thread=56 (was 48) Potentially hanging thread: AsyncFSWAL-0-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testLastParentCleanedEvenIfDaughterGoneFirst/MasterData-prefix:mockmaster.example.org,1234,-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PEWorker-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167) app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2176) Potentially hanging thread: PEWorker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167) app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2176) Potentially hanging thread: ProcExecTimeout java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:274) app//org.apache.hadoop.hbase.procedure2.util.DelayedUtil.takeWithoutInterrupt(DelayedUtil.java:81) app//org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.run(TimeoutExecutorThread.java:56) Potentially hanging thread: AsyncFSOutputFlusher-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testLastParentCleanedEvenIfDaughterGoneFirst/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731208425993 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WorkerMonitor java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:279) app//org.apache.hadoop.hbase.procedure2.util.DelayedUtil.takeWithoutInterrupt(DelayedUtil.java:81) app//org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.run(TimeoutExecutorThread.java:56) - Thread LEAK? -, OpenFileDescriptor=533 (was 501) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=866 (was 866), ProcessCount=12 (was 12), AvailableMemoryMB=1440 (was 1525) 2024-11-10T03:13:49,248 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.janitor.TestCatalogJanitor#testSplitParentFirstComparator Thread=51, OpenFileDescriptor=535, MaxFileDescriptor=1048576, SystemLoadAverage=866, ProcessCount=12, AvailableMemoryMB=1440 2024-11-10T03:13:49,274 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testLastParentCleanedEvenIfDaughterGoneFirst/data/default/testLastParentCleanedEvenIfDaughterGoneFirst/7d633c6c1926c131562f497764056809 2024-11-10T03:13:49,293 INFO [Time-limited test {}] util.FSUtils(489): Created version file at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testSplitParentFirstComparator with version=8 2024-11-10T03:13:49,294 DEBUG [Time-limited test {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testSplitParentFirstComparator/hbase.id] with ID: 504ac898-0e5b-4393-a1c3-30e425d9c8da 2024-11-10T03:13:49,294 DEBUG [Time-limited test {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testSplitParentFirstComparator/.tmp/hbase.id 2024-11-10T03:13:49,317 DEBUG [Time-limited test {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testSplitParentFirstComparator/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testSplitParentFirstComparator/hbase.id] 2024-11-10T03:13:49,389 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T03:13:49,390 INFO [Time-limited test {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T03:13:49,391 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T03:13:49,392 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T03:13:49,420 INFO [Time-limited test {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testSplitParentFirstComparator/MasterData/data/master/store 2024-11-10T03:13:49,429 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:13:49,429 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T03:13:49,429 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:49,429 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:49,429 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T03:13:49,430 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:49,430 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:49,430 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731208429429Disabling compacts and flushes for region at 1731208429429Disabling writes for close at 1731208429429Writing region close event to WAL at 1731208429430 (+1 ms)Closed at 1731208429430 2024-11-10T03:13:49,430 WARN [Time-limited test {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testSplitParentFirstComparator/MasterData/data/master/store/.initializing 2024-11-10T03:13:49,430 DEBUG [Time-limited test {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testSplitParentFirstComparator/MasterData/WALs/mockmaster.example.org,1234,-1 2024-11-10T03:13:49,435 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T03:13:49,438 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=mockmaster.example.org%2C1234%2C-1, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testSplitParentFirstComparator/MasterData/WALs/mockmaster.example.org,1234,-1, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testSplitParentFirstComparator/MasterData/oldWALs, maxLogs=10 2024-11-10T03:13:49,477 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testSplitParentFirstComparator/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731208429439 2024-11-10T03:13:49,481 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T03:13:49,481 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T03:13:49,482 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:13:49,482 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:49,482 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:49,485 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:49,493 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T03:13:49,493 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:49,493 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T03:13:49,494 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:49,498 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T03:13:49,498 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:49,499 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:13:49,499 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:49,508 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T03:13:49,508 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:49,509 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:13:49,509 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:49,517 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T03:13:49,517 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:49,518 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:13:49,518 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:49,519 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testSplitParentFirstComparator/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:49,519 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testSplitParentFirstComparator/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:49,520 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:49,520 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:49,520 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T03:13:49,521 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:49,556 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testSplitParentFirstComparator/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T03:13:49,557 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70445919, jitterRate=0.04972599446773529}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T03:13:49,557 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731208429483Initializing all the Stores at 1731208429483Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731208429483Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208429484 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208429485 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208429485Cleaning up temporary data from old regions at 1731208429520 (+35 ms)Region opened successfully at 1731208429557 (+37 ms) 2024-11-10T03:13:49,558 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T03:13:49,569 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T03:13:49,569 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered NoopProcedureStore lease in 0 msec 2024-11-10T03:13:49,569 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded NoopProcedureStore in 0 msec 2024-11-10T03:13:49,584 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731208459584 2024-11-10T03:13:49,584 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T03:13:49,588 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,100,1 2024-11-10T03:13:49,588 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,101,1 2024-11-10T03:13:49,588 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,102,1 2024-11-10T03:13:49,588 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,103,1 2024-11-10T03:13:49,589 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,104,1 2024-11-10T03:13:49,589 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,105,1 2024-11-10T03:13:49,589 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,106,1 2024-11-10T03:13:49,589 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,107,1 2024-11-10T03:13:49,589 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,108,1 2024-11-10T03:13:49,589 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,109,1 2024-11-10T03:13:49,590 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T03:13:49,590 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T03:13:49,591 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T03:13:49,602 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.janitor.TestCatalogJanitor#testSplitParentFirstComparator Thread=56 (was 51) Potentially hanging thread: PEWorker-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167) app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2176) Potentially hanging thread: AsyncFSOutputFlusher-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testSplitParentFirstComparator/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731208429439 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PEWorker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167) app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2176) Potentially hanging thread: AsyncFSWAL-0-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testSplitParentFirstComparator/MasterData-prefix:mockmaster.example.org,1234,-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=559 (was 535) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=866 (was 866), ProcessCount=11 (was 12), AvailableMemoryMB=1431 (was 1440) 2024-11-10T03:13:49,611 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.janitor.TestCatalogJanitor#testDontCleanMergeIfFileSystemException Thread=54, OpenFileDescriptor=561, MaxFileDescriptor=1048576, SystemLoadAverage=866, ProcessCount=11, AvailableMemoryMB=1431 2024-11-10T03:13:49,633 INFO [Time-limited test {}] util.FSUtils(489): Created version file at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDontCleanMergeIfFileSystemException with version=8 2024-11-10T03:13:49,634 DEBUG [Time-limited test {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDontCleanMergeIfFileSystemException/hbase.id] with ID: c08dbaf8-adc8-4828-afe2-dbbe7db35401 2024-11-10T03:13:49,634 DEBUG [Time-limited test {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDontCleanMergeIfFileSystemException/.tmp/hbase.id 2024-11-10T03:13:49,645 DEBUG [Time-limited test {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDontCleanMergeIfFileSystemException/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDontCleanMergeIfFileSystemException/hbase.id] 2024-11-10T03:13:49,768 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T03:13:49,769 INFO [Time-limited test {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T03:13:49,770 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T03:13:49,773 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T03:13:49,897 INFO [Time-limited test {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDontCleanMergeIfFileSystemException/MasterData/data/master/store 2024-11-10T03:13:49,930 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:13:49,930 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T03:13:49,930 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:49,930 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:49,930 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T03:13:49,930 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:49,930 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:49,931 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731208429930Disabling compacts and flushes for region at 1731208429930Disabling writes for close at 1731208429930Writing region close event to WAL at 1731208429930Closed at 1731208429930 2024-11-10T03:13:49,931 WARN [Time-limited test {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDontCleanMergeIfFileSystemException/MasterData/data/master/store/.initializing 2024-11-10T03:13:49,931 DEBUG [Time-limited test {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDontCleanMergeIfFileSystemException/MasterData/WALs/mockmaster.example.org,1234,-1 2024-11-10T03:13:49,948 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T03:13:49,962 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=mockmaster.example.org%2C1234%2C-1, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDontCleanMergeIfFileSystemException/MasterData/WALs/mockmaster.example.org,1234,-1, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDontCleanMergeIfFileSystemException/MasterData/oldWALs, maxLogs=10 2024-11-10T03:13:50,010 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDontCleanMergeIfFileSystemException/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731208429962 2024-11-10T03:13:50,013 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T03:13:50,013 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T03:13:50,013 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:13:50,013 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:50,013 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:50,016 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:50,034 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T03:13:50,034 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:50,034 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T03:13:50,036 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:50,048 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T03:13:50,048 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:50,049 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:13:50,049 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:50,058 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T03:13:50,059 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:50,059 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:13:50,060 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:50,072 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T03:13:50,072 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:50,073 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:13:50,075 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:50,076 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDontCleanMergeIfFileSystemException/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:50,076 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDontCleanMergeIfFileSystemException/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:50,076 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:50,076 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:50,077 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T03:13:50,077 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:50,123 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDontCleanMergeIfFileSystemException/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T03:13:50,124 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75331139, jitterRate=0.12252144515514374}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T03:13:50,125 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731208430014Initializing all the Stores at 1731208430014Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731208430014Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208430016 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208430016Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208430016Cleaning up temporary data from old regions at 1731208430076 (+60 ms)Region opened successfully at 1731208430124 (+48 ms) 2024-11-10T03:13:50,129 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T03:13:50,144 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T03:13:50,144 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered NoopProcedureStore lease in 0 msec 2024-11-10T03:13:50,144 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded NoopProcedureStore in 0 msec 2024-11-10T03:13:50,163 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731208460163 2024-11-10T03:13:50,163 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T03:13:50,175 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,100,1 2024-11-10T03:13:50,176 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,101,1 2024-11-10T03:13:50,176 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,102,1 2024-11-10T03:13:50,176 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,103,1 2024-11-10T03:13:50,177 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,104,1 2024-11-10T03:13:50,177 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,105,1 2024-11-10T03:13:50,177 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,106,1 2024-11-10T03:13:50,178 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,107,1 2024-11-10T03:13:50,178 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,108,1 2024-11-10T03:13:50,178 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,109,1 2024-11-10T03:13:50,181 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:50,687 DEBUG [Time-limited test {}] janitor.CatalogJanitor(258): Cleaning merged region {ENCODED => ffbf61a066b1c1ae2b089c9ec4ee7797, NAME => 'testDontCleanMergeIfFileSystemException,aaa,1731208430181.ffbf61a066b1c1ae2b089c9ec4ee7797.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T03:13:50,690 ERROR [Time-limited test {}] janitor.CatalogJanitor(420): Error trying to determine if region exists, assuming exists and has references java.io.IOException: Some exception at org.apache.hadoop.hbase.util.CommonFSUtils.isExists(CommonFSUtils.java:673) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.janitor.CatalogJanitor.checkRegionReferences(CatalogJanitor.java:416) ~[classes/:?] at org.apache.hadoop.hbase.master.janitor.CatalogJanitor.cleanMergeRegion(CatalogJanitor.java:262) ~[classes/:?] at org.apache.hadoop.hbase.master.janitor.TestCatalogJanitor.testDontCleanMergeIfFileSystemException(TestCatalogJanitor.java:201) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:13:50,691 DEBUG [Time-limited test {}] janitor.CatalogJanitor(283): Deferring cleanup up of 2 parents of merged region ffbf61a066b1c1ae2b089c9ec4ee7797, because references still exist in merged region or we encountered an exception in checking 2024-11-10T03:13:50,692 DEBUG [Time-limited test {}] janitor.CatalogJanitor(258): Cleaning merged region {ENCODED => ffbf61a066b1c1ae2b089c9ec4ee7797, NAME => 'testDontCleanMergeIfFileSystemException,aaa,1731208430181.ffbf61a066b1c1ae2b089c9ec4ee7797.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T03:13:50,695 ERROR [Time-limited test {}] janitor.CatalogJanitor(441): Error trying to determine if region ffbf61a066b1c1ae2b089c9ec4ee7797 has references, assuming it does java.io.IOException: Some exception at org.apache.hadoop.hbase.master.janitor.TestCatalogJanitor.lambda$testDontCleanMergeIfFileSystemException$0(TestCatalogJanitor.java:211) ~[test-classes/:?] at org.mockito.internal.stubbing.StubbedInvocationMatcher.answer(StubbedInvocationMatcher.java:42) ~[mockito-core-4.11.0.jar:?] at org.mockito.internal.handler.MockHandlerImpl.handle(MockHandlerImpl.java:103) ~[mockito-core-4.11.0.jar:?] at org.mockito.internal.handler.NullResultGuardian.handle(NullResultGuardian.java:29) ~[mockito-core-4.11.0.jar:?] at org.mockito.internal.handler.InvocationNotifierHandler.handle(InvocationNotifierHandler.java:34) ~[mockito-core-4.11.0.jar:?] at org.mockito.internal.creation.bytebuddy.MockMethodInterceptor.doIntercept(MockMethodInterceptor.java:82) ~[mockito-core-4.11.0.jar:?] at org.mockito.internal.creation.bytebuddy.MockMethodInterceptor.doIntercept(MockMethodInterceptor.java:56) ~[mockito-core-4.11.0.jar:?] at org.mockito.internal.creation.bytebuddy.MockMethodInterceptor$DispatcherDefaultingToRealMethod.interceptSuperCallable(MockMethodInterceptor.java:145) ~[mockito-core-4.11.0.jar:?] at org.apache.hadoop.fs.LocalFileSystem$MockitoMock$PXtwpsiC.exists(Unknown Source) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.HRegionFileSystem.openRegionFromFileSystem(HRegionFileSystem.java:959) ~[classes/:?] at org.apache.hadoop.hbase.master.janitor.CatalogJanitor.checkRegionReferences(CatalogJanitor.java:428) ~[classes/:?] at org.apache.hadoop.hbase.master.janitor.CatalogJanitor.cleanMergeRegion(CatalogJanitor.java:262) ~[classes/:?] at org.apache.hadoop.hbase.master.janitor.TestCatalogJanitor.testDontCleanMergeIfFileSystemException(TestCatalogJanitor.java:214) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T03:13:50,698 DEBUG [Time-limited test {}] janitor.CatalogJanitor(283): Deferring cleanup up of 2 parents of merged region ffbf61a066b1c1ae2b089c9ec4ee7797, because references still exist in merged region or we encountered an exception in checking 2024-11-10T03:13:50,698 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T03:13:50,698 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T03:13:50,698 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T03:13:50,729 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.janitor.TestCatalogJanitor#testDontCleanMergeIfFileSystemException Thread=60 (was 54) Potentially hanging thread: AsyncFSOutputFlusher-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDontCleanMergeIfFileSystemException/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731208429962 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PEWorker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167) app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2176) Potentially hanging thread: AsyncFSWAL-0-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testDontCleanMergeIfFileSystemException/MasterData-prefix:mockmaster.example.org,1234,-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureDispatcherTimeoutThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.procedure2.util.DelayedUtil.takeWithoutInterrupt(DelayedUtil.java:81) app//org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher$TimeoutExecutorThread.run(RemoteProcedureDispatcher.java:328) - Thread LEAK? -, OpenFileDescriptor=593 (was 561) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=877 (was 866) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1370 (was 1431) 2024-11-10T03:13:50,750 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.janitor.TestCatalogJanitor#testArchiveOldRegion Thread=58, OpenFileDescriptor=593, MaxFileDescriptor=1048576, SystemLoadAverage=877, ProcessCount=11, AvailableMemoryMB=1370 2024-11-10T03:13:50,817 INFO [Time-limited test {}] util.FSUtils(489): Created version file at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion with version=8 2024-11-10T03:13:50,819 DEBUG [Time-limited test {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/hbase.id] with ID: fb96b33c-0ddb-4ade-9f42-c635f243d895 2024-11-10T03:13:50,819 DEBUG [Time-limited test {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/.tmp/hbase.id 2024-11-10T03:13:50,862 DEBUG [Time-limited test {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/hbase.id] 2024-11-10T03:13:50,943 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T03:13:50,944 INFO [Time-limited test {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T03:13:50,945 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T03:13:50,946 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T03:13:50,986 INFO [Time-limited test {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/MasterData/data/master/store 2024-11-10T03:13:51,030 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:13:51,031 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T03:13:51,031 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:51,031 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:51,031 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T03:13:51,031 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:51,031 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T03:13:51,031 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731208431031Disabling compacts and flushes for region at 1731208431031Disabling writes for close at 1731208431031Writing region close event to WAL at 1731208431031Closed at 1731208431031 2024-11-10T03:13:51,031 WARN [Time-limited test {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/MasterData/data/master/store/.initializing 2024-11-10T03:13:51,031 DEBUG [Time-limited test {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/MasterData/WALs/mockmaster.example.org,1234,-1 2024-11-10T03:13:51,047 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T03:13:51,061 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=mockmaster.example.org%2C1234%2C-1, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/MasterData/WALs/mockmaster.example.org,1234,-1, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/MasterData/oldWALs, maxLogs=10 2024-11-10T03:13:51,116 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731208431061 2024-11-10T03:13:51,118 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T03:13:51,118 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T03:13:51,120 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T03:13:51,121 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:51,121 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:51,128 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:51,135 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T03:13:51,136 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:51,136 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T03:13:51,136 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:51,172 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T03:13:51,172 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:51,172 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:13:51,173 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:51,193 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T03:13:51,193 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:51,193 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:13:51,194 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:51,203 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T03:13:51,203 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:51,203 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T03:13:51,203 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:51,204 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:51,204 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:51,209 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:51,209 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:51,209 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T03:13:51,210 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T03:13:51,252 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T03:13:51,253 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75050975, jitterRate=0.1183466762304306}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T03:13:51,253 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731208431121Initializing all the Stores at 1731208431121Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731208431122 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208431124 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208431124Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731208431124Cleaning up temporary data from old regions at 1731208431209 (+85 ms)Region opened successfully at 1731208431253 (+44 ms) 2024-11-10T03:13:51,260 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T03:13:51,282 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T03:13:51,282 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered NoopProcedureStore lease in 0 msec 2024-11-10T03:13:51,282 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded NoopProcedureStore in 0 msec 2024-11-10T03:13:51,343 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731208461343 2024-11-10T03:13:51,344 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T03:13:51,345 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,100,1 2024-11-10T03:13:51,345 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,101,1 2024-11-10T03:13:51,345 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,102,1 2024-11-10T03:13:51,346 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,103,1 2024-11-10T03:13:51,348 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,104,1 2024-11-10T03:13:51,348 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,105,1 2024-11-10T03:13:51,348 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,106,1 2024-11-10T03:13:51,348 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,107,1 2024-11-10T03:13:51,349 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,108,1 2024-11-10T03:13:51,349 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,109,1 2024-11-10T03:13:51,350 DEBUG [Time-limited test {}] janitor.TestCatalogJanitor(566): Table dir:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/data/default/testArchiveOldRegion 2024-11-10T03:13:51,350 DEBUG [Time-limited test {}] janitor.TestCatalogJanitor(567): Store dir:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/data/default/testArchiveOldRegion/c74d447f4986c018787b4b997df44d32/cf 2024-11-10T03:13:51,350 DEBUG [Time-limited test {}] janitor.TestCatalogJanitor(568): Store archive dir:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/archive/data/default/testArchiveOldRegion/c74d447f4986c018787b4b997df44d32/cf 2024-11-10T03:13:51,489 DEBUG [Time-limited test {}] janitor.TestCatalogJanitor(711): Adding 2 store files to the storedir:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/data/default/testArchiveOldRegion/c74d447f4986c018787b4b997df44d32/cf 2024-11-10T03:13:51,490 DEBUG [Time-limited test {}] janitor.TestCatalogJanitor(576): Have store file:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/data/default/testArchiveOldRegion/c74d447f4986c018787b4b997df44d32/cf/_store0 2024-11-10T03:13:51,490 DEBUG [Time-limited test {}] janitor.TestCatalogJanitor(576): Have store file:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/data/default/testArchiveOldRegion/c74d447f4986c018787b4b997df44d32/cf/_store1 2024-11-10T03:13:51,490 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => c74d447f4986c018787b4b997df44d32, NAME => 'testArchiveOldRegion,aaa,1731208431349.c74d447f4986c018787b4b997df44d32.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T03:13:51,491 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T03:13:51,492 DEBUG [Time-limited test {}] janitor.CatalogJanitor(351): Deleting region c74d447f4986c018787b4b997df44d32 because daughters -- c74d447f4986c018787b4b997df44d32, 28ea7fbcbf4c07aa72f67a5782d55452 -- no longer hold references 2024-11-10T03:13:51,492 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testArchiveOldRegion, region=c74d447f4986c018787b4b997df44d32 2024-11-10T03:13:51,492 DEBUG [Time-limited test {}] janitor.CatalogJanitor(358): Submitted procedure pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testArchiveOldRegion, region=c74d447f4986c018787b4b997df44d32 for split parent {ENCODED => c74d447f4986c018787b4b997df44d32, NAME => 'testArchiveOldRegion,aaa,1731208431349.c74d447f4986c018787b4b997df44d32.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T03:13:51,492 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testArchiveOldRegion, region=c74d447f4986c018787b4b997df44d32 2024-11-10T03:13:51,493 DEBUG [PEWorker-1 {}] assignment.GCRegionProcedure(87): Archiving region=c74d447f4986c018787b4b997df44d32 2024-11-10T03:13:51,493 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/data/default/testArchiveOldRegion/c74d447f4986c018787b4b997df44d32 2024-11-10T03:13:51,493 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/data/default/testArchiveOldRegion/c74d447f4986c018787b4b997df44d32/cf] 2024-11-10T03:13:51,565 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/data/default/testArchiveOldRegion/c74d447f4986c018787b4b997df44d32/cf/_store0 to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/archive/data/default/testArchiveOldRegion/c74d447f4986c018787b4b997df44d32/cf/_store0 2024-11-10T03:13:51,565 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/data/default/testArchiveOldRegion/c74d447f4986c018787b4b997df44d32/cf/_store1 to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/archive/data/default/testArchiveOldRegion/c74d447f4986c018787b4b997df44d32/cf/_store1 2024-11-10T03:13:51,566 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/data/default/testArchiveOldRegion/c74d447f4986c018787b4b997df44d32 2024-11-10T03:13:51,569 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testArchiveOldRegion,aaa,1731208431349.c74d447f4986c018787b4b997df44d32.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731208431566"}]},"ts":"9223372036854775807"} 2024-11-10T03:13:51,569 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T03:13:51,569 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c74d447f4986c018787b4b997df44d32, NAME => 'testArchiveOldRegion,aaa,1731208431349.c74d447f4986c018787b4b997df44d32.', STARTKEY => 'aaa', ENDKEY => 'eee'}] 2024-11-10T03:13:51,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; GCRegionProcedure table=testArchiveOldRegion, region=c74d447f4986c018787b4b997df44d32 in 77 msec 2024-11-10T03:13:51,742 DEBUG [Time-limited test {}] janitor.TestCatalogJanitor(588): Finished cleanup of parent region 2024-11-10T03:13:51,743 DEBUG [Time-limited test {}] janitor.TestCatalogJanitor(606): Current archived files: 2024-11-10T03:13:51,743 DEBUG [Time-limited test {}] janitor.TestCatalogJanitor(608): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/data/default/testArchiveOldRegion/c74d447f4986c018787b4b997df44d32/cf/_store0 2024-11-10T03:13:51,743 DEBUG [Time-limited test {}] janitor.TestCatalogJanitor(608): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/data/default/testArchiveOldRegion/c74d447f4986c018787b4b997df44d32/cf/_store1 2024-11-10T03:13:51,743 DEBUG [Time-limited test {}] janitor.TestCatalogJanitor(606): Current archived files: 2024-11-10T03:13:51,744 DEBUG [Time-limited test {}] janitor.TestCatalogJanitor(608): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/archive/data/default/testArchiveOldRegion/c74d447f4986c018787b4b997df44d32/cf/_store0 2024-11-10T03:13:51,744 DEBUG [Time-limited test {}] janitor.TestCatalogJanitor(608): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/archive/data/default/testArchiveOldRegion/c74d447f4986c018787b4b997df44d32/cf/_store1 2024-11-10T03:13:51,748 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T03:13:51,748 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T03:13:51,748 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T03:13:51,757 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.janitor.TestCatalogJanitor#testArchiveOldRegion Thread=62 (was 58) Potentially hanging thread: AsyncFSOutputFlusher-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731208431061 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ffddb2d-fe20-c22b-3f45-10836e107d79/testArchiveOldRegion/MasterData-prefix:mockmaster.example.org,1234,-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=627 (was 593) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=877 (was 877), ProcessCount=11 (was 11), AvailableMemoryMB=1321 (was 1370)