2025-01-10 07:53:42,842 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@183ec003 2025-01-10 07:53:42,856 main DEBUG Took 0.011977 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2025-01-10 07:53:42,856 main DEBUG PluginManager 'Core' found 129 plugins 2025-01-10 07:53:42,856 main DEBUG PluginManager 'Level' found 0 plugins 2025-01-10 07:53:42,857 main DEBUG PluginManager 'Lookup' found 16 plugins 2025-01-10 07:53:42,859 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-10 07:53:42,870 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2025-01-10 07:53:42,891 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-10 07:53:42,892 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-10 07:53:42,893 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-10 07:53:42,893 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-10 07:53:42,894 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-10 07:53:42,895 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-10 07:53:42,896 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-10 07:53:42,896 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-10 07:53:42,896 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-10 07:53:42,897 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-10 07:53:42,897 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-10 07:53:42,897 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-10 07:53:42,898 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-10 07:53:42,898 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-10 07:53:42,899 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-10 07:53:42,899 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-10 07:53:42,899 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-10 07:53:42,899 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-10 07:53:42,900 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-10 07:53:42,900 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-10 07:53:42,900 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-10 07:53:42,901 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-10 07:53:42,901 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-10 07:53:42,901 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2025-01-10 07:53:42,902 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-10 07:53:42,902 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2025-01-10 07:53:42,903 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2025-01-10 07:53:42,904 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2025-01-10 07:53:42,906 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2025-01-10 07:53:42,906 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2025-01-10 07:53:42,907 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2025-01-10 07:53:42,907 main DEBUG PluginManager 'Converter' found 47 plugins 2025-01-10 07:53:42,915 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2025-01-10 07:53:42,917 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2025-01-10 07:53:42,919 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2025-01-10 07:53:42,919 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2025-01-10 07:53:42,919 main DEBUG createAppenders(={Console}) 2025-01-10 07:53:42,920 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@183ec003 initialized 2025-01-10 07:53:42,920 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@183ec003 2025-01-10 07:53:42,920 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@183ec003 OK. 2025-01-10 07:53:42,921 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2025-01-10 07:53:42,921 main DEBUG OutputStream closed 2025-01-10 07:53:42,921 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2025-01-10 07:53:42,921 main DEBUG Appender DefaultConsole-1 stopped with status true 2025-01-10 07:53:42,922 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4bb33f74 OK 2025-01-10 07:53:42,981 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2025-01-10 07:53:42,983 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2025-01-10 07:53:42,984 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2025-01-10 07:53:42,985 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2025-01-10 07:53:42,985 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2025-01-10 07:53:42,986 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2025-01-10 07:53:42,986 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2025-01-10 07:53:42,986 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2025-01-10 07:53:42,986 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2025-01-10 07:53:42,987 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2025-01-10 07:53:42,987 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2025-01-10 07:53:42,987 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2025-01-10 07:53:42,988 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2025-01-10 07:53:42,988 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2025-01-10 07:53:42,988 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2025-01-10 07:53:42,988 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2025-01-10 07:53:42,988 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2025-01-10 07:53:42,989 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2025-01-10 07:53:42,991 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2025-01-10 07:53:42,991 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@3abada5a) with optional ClassLoader: null 2025-01-10 07:53:42,992 main DEBUG Shutdown hook enabled. Registering a new one. 2025-01-10 07:53:42,992 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@3abada5a] started OK. 2025-01-10T07:53:43,006 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.bucket.TestVerifyBucketCacheFile timeout: 26 mins 2025-01-10 07:53:43,009 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2025-01-10 07:53:43,009 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2025-01-10T07:53:43,050 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestVerifyBucketCacheFile#testBucketCacheRecovery[0: blockSize=8,192, bucketSizes=null] Thread=11, OpenFileDescriptor=284, MaxFileDescriptor=1048576, SystemLoadAverage=558, ProcessCount=11, AvailableMemoryMB=6657 2025-01-10T07:53:43,312 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab 2025-01-10T07:53:43,386 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2025-01-10T07:53:43,466 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-10T07:53:43,504 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/bucket.cache 2025-01-10T07:53:43,505 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:43,505 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:43,506 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 9223372036854775807 2025-01-10T07:53:43,508 INFO [Thread-5 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:43,508 WARN [Thread-5 {}] bucket.BucketCache(1520): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2025-01-10T07:53:43,509 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/bucket.cache, capacity=32 MB, blockSize=8 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/bucket.persistence1736495623494, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:43,519 INFO [Thread-5 {}] bucket.BucketAllocator(343): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:43,519 INFO [Thread-5 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/bucket.persistence1736495623494 is complete. 2025-01-10T07:53:43,628 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2025-01-10T07:53:43,962 DEBUG [Time-limited test {}] bucket.BucketCache(1474): Thread Time-limited test started persisting bucket cache to file 2025-01-10T07:53:43,962 DEBUG [Time-limited test {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:43,962 DEBUG [Time-limited test {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 3, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:44,080 DEBUG [Time-limited test {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 3, fullycachedFiles size: 0 2025-01-10T07:53:44,080 DEBUG [Time-limited test {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 3, fullycachedFiles size: 0, file name: bucket.persistence17364956234941736495623962 2025-01-10T07:53:44,080 DEBUG [Time-limited test {}] bucket.BucketCache(1496): Thread Time-limited test finished persisting bucket cache to file, renaming 2025-01-10T07:53:44,082 DEBUG [Time-limited test {}] bucket.BucketCache(843): removed key -8027807471553991689_0 from back map with offset lock 4193280 in the evict process 2025-01-10T07:53:44,183 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/bucket.cache 2025-01-10T07:53:44,184 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:44,184 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:44,184 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 9223372036854775807 2025-01-10T07:53:44,184 INFO [Thread-7 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:44,184 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/bucket.cache, capacity=32 MB, blockSize=8 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/bucket.persistence1736495623494, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:44,190 INFO [Thread-7 {}] bucket.BucketCache(1540): Reading new chunked format of persistence. 2025-01-10T07:53:44,192 INFO [Thread-7 {}] bucket.BucketCache(1693): Retrieved 1 of chunks with blockCount = 3. 2025-01-10T07:53:44,196 WARN [Thread-7 {}] bucket.BucketCache(1601): Checksum for cache file failed. We need to validate each cache key in the backing map. This may take some time, so we'll do it in a background thread, 2025-01-10T07:53:44,196 DEBUG [Thread-7 {}] bucket.BucketCache(1557): Updating region size map after retrieving cached file list 2025-01-10T07:53:44,197 INFO [Thread-7 {}] bucket.BucketAllocator(343): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:44,197 INFO [Thread-7 {}] bucket.BucketCache(1550): Bucket cache retrieved from file successfully with size: 3 2025-01-10T07:53:44,197 INFO [Thread-7 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/bucket.persistence1736495623494 is complete. 2025-01-10T07:53:44,247 DEBUG [Thread-9 {}] bucket.BucketCache(1619): Check for key -8027807471553991689_0 failed. Evicting. 2025-01-10T07:53:44,247 DEBUG [Thread-9 {}] bucket.BucketCache(843): removed key -8027807471553991689_0 from back map with offset lock 4193280 in the evict process 2025-01-10T07:53:44,247 INFO [Thread-9 {}] bucket.BucketCache(1625): Finished validating 3 keys in the backing map. Recovered: 2. This took 1ms. 2025-01-10T07:53:44,307 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestVerifyBucketCacheFile#testBucketCacheRecovery[0: blockSize=8,192, bucketSizes=null] Thread=23 (was 11) Potentially hanging thread: Time-limited test-BucketCacheWriter-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420) app//org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getRAMQueueEntries(BucketCache.java:1463) app//org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread.run(BucketCache.java:1244) Potentially hanging thread: BucketCacheStatsExecutor java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test-BucketCacheWriter-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420) app//org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getRAMQueueEntries(BucketCache.java:1463) app//org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread.run(BucketCache.java:1244) Potentially hanging thread: BucketCacheStatsExecutor java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test-BucketCacheWriter-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420) app//org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getRAMQueueEntries(BucketCache.java:1463) app//org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread.run(BucketCache.java:1244) Potentially hanging thread: Time-limited test-BucketCacheWriter-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420) app//org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getRAMQueueEntries(BucketCache.java:1463) app//org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread.run(BucketCache.java:1244) Potentially hanging thread: bucket-cache-persister java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) Potentially hanging thread: bucket-cache-persister java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) Potentially hanging thread: Time-limited test-BucketCacheWriter-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420) app//org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getRAMQueueEntries(BucketCache.java:1463) app//org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread.run(BucketCache.java:1244) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test-BucketCacheWriter-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420) app//org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getRAMQueueEntries(BucketCache.java:1463) app//org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread.run(BucketCache.java:1244) - Thread LEAK? -, OpenFileDescriptor=288 (was 284) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=558 (was 558), ProcessCount=11 (was 11), AvailableMemoryMB=6600 (was 6657) 2025-01-10T07:53:44,313 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestVerifyBucketCacheFile#testRetrieveFromFile[0: blockSize=8,192, bucketSizes=null] Thread=23, OpenFileDescriptor=288, MaxFileDescriptor=1048576, SystemLoadAverage=558, ProcessCount=11, AvailableMemoryMB=6600 2025-01-10T07:53:44,326 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-log-dir so I do NOT create it in target/test-data/d6bd01cd-a623-1d16-5ab4-57fa19bd0211 2025-01-10T07:53:44,327 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.log.dir property value differs in configuration and system: Configuration=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/../logs while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-log-dir Erasing configuration value by system value. 2025-01-10T07:53:44,327 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-tmp-dir so I do NOT create it in target/test-data/d6bd01cd-a623-1d16-5ab4-57fa19bd0211 2025-01-10T07:53:44,327 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-tmp-dir Erasing configuration value by system value. 2025-01-10T07:53:44,327 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d6bd01cd-a623-1d16-5ab4-57fa19bd0211 2025-01-10T07:53:44,328 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-10T07:53:44,350 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d6bd01cd-a623-1d16-5ab4-57fa19bd0211/bucket.cache 2025-01-10T07:53:44,351 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:44,351 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:44,351 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2025-01-10T07:53:44,352 INFO [Thread-11 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:44,352 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d6bd01cd-a623-1d16-5ab4-57fa19bd0211/bucket.cache, capacity=32 MB, blockSize=8 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d6bd01cd-a623-1d16-5ab4-57fa19bd0211/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:44,352 WARN [Thread-11 {}] bucket.BucketCache(1520): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2025-01-10T07:53:44,352 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:44,353 INFO [Thread-11 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d6bd01cd-a623-1d16-5ab4-57fa19bd0211/bucket.persistence is complete. 2025-01-10T07:53:44,553 INFO [Time-limited test {}] bucket.BucketCache(1726): Disabling cache 2025-01-10T07:53:44,553 INFO [Time-limited test {}] bucket.BucketCache(1750): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d6bd01cd-a623-1d16-5ab4-57fa19bd0211/bucket.persistence 2025-01-10T07:53:44,554 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2025-01-10T07:53:44,554 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2025-01-10T07:53:44,554 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2025-01-10T07:53:44,554 INFO [Time-limited test {}] bucket.BucketCache(1756): Shutting down cache persister thread. 2025-01-10T07:53:44,554 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2025-01-10T07:53:44,564 DEBUG [Time-limited test {}] bucket.BucketCache(1474): Thread Time-limited test started persisting bucket cache to file 2025-01-10T07:53:44,565 DEBUG [Time-limited test {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:44,565 DEBUG [Time-limited test {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:44,568 DEBUG [Time-limited test {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2025-01-10T07:53:44,568 DEBUG [Time-limited test {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1736495624565 2025-01-10T07:53:44,568 DEBUG [Time-limited test {}] bucket.BucketCache(1496): Thread Time-limited test finished persisting bucket cache to file, renaming 2025-01-10T07:53:44,582 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d6bd01cd-a623-1d16-5ab4-57fa19bd0211/bucket.cache 2025-01-10T07:53:44,582 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:44,582 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:44,583 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2025-01-10T07:53:44,583 INFO [Thread-13 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:44,583 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d6bd01cd-a623-1d16-5ab4-57fa19bd0211/bucket.cache, capacity=32 MB, blockSize=8 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d6bd01cd-a623-1d16-5ab4-57fa19bd0211/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:44,583 INFO [Thread-13 {}] bucket.BucketCache(1540): Reading new chunked format of persistence. 2025-01-10T07:53:44,584 INFO [Thread-13 {}] bucket.BucketCache(1693): Retrieved 1 of chunks with blockCount = 1. 2025-01-10T07:53:44,587 DEBUG [Thread-13 {}] bucket.BucketCache(1557): Updating region size map after retrieving cached file list 2025-01-10T07:53:44,587 INFO [Thread-13 {}] bucket.BucketAllocator(343): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:44,588 INFO [Thread-13 {}] bucket.BucketCache(1550): Bucket cache retrieved from file successfully with size: 1 2025-01-10T07:53:44,588 INFO [Thread-13 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d6bd01cd-a623-1d16-5ab4-57fa19bd0211/bucket.persistence is complete. 2025-01-10T07:53:44,683 INFO [Time-limited test {}] bucket.BucketCache(1726): Disabling cache 2025-01-10T07:53:44,684 INFO [Time-limited test {}] bucket.BucketCache(1750): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d6bd01cd-a623-1d16-5ab4-57fa19bd0211/bucket.persistence 2025-01-10T07:53:44,684 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2025-01-10T07:53:44,684 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2025-01-10T07:53:44,684 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2025-01-10T07:53:44,684 INFO [Time-limited test {}] bucket.BucketCache(1756): Shutting down cache persister thread. 2025-01-10T07:53:44,684 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2025-01-10T07:53:44,694 DEBUG [Time-limited test {}] bucket.BucketCache(1474): Thread Time-limited test started persisting bucket cache to file 2025-01-10T07:53:44,695 DEBUG [Time-limited test {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:44,695 DEBUG [Time-limited test {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:44,698 DEBUG [Time-limited test {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2025-01-10T07:53:44,698 DEBUG [Time-limited test {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1736495624694 2025-01-10T07:53:44,698 DEBUG [Time-limited test {}] bucket.BucketCache(1496): Thread Time-limited test finished persisting bucket cache to file, renaming 2025-01-10T07:53:44,714 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d6bd01cd-a623-1d16-5ab4-57fa19bd0211/bucket.cache 2025-01-10T07:53:44,714 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:44,714 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:44,715 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2025-01-10T07:53:44,715 INFO [Thread-16 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:44,715 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d6bd01cd-a623-1d16-5ab4-57fa19bd0211/bucket.cache, capacity=32 MB, blockSize=8 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d6bd01cd-a623-1d16-5ab4-57fa19bd0211/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:44,715 INFO [Thread-16 {}] bucket.BucketCache(1540): Reading new chunked format of persistence. 2025-01-10T07:53:44,716 INFO [Thread-16 {}] bucket.BucketCache(1693): Retrieved 1 of chunks with blockCount = 1. 2025-01-10T07:53:44,719 WARN [Thread-16 {}] bucket.BucketCache(1601): Checksum for cache file failed. We need to validate each cache key in the backing map. This may take some time, so we'll do it in a background thread, 2025-01-10T07:53:44,719 DEBUG [Thread-16 {}] bucket.BucketCache(1557): Updating region size map after retrieving cached file list 2025-01-10T07:53:44,719 INFO [Thread-16 {}] bucket.BucketAllocator(343): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:44,719 INFO [Thread-16 {}] bucket.BucketCache(1550): Bucket cache retrieved from file successfully with size: 1 2025-01-10T07:53:44,719 INFO [Thread-16 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d6bd01cd-a623-1d16-5ab4-57fa19bd0211/bucket.persistence is complete. 2025-01-10T07:53:44,769 DEBUG [Thread-18 {}] bucket.BucketCache(1619): Check for key 2307686003434224477_0 failed. Evicting. 2025-01-10T07:53:44,769 DEBUG [Thread-18 {}] bucket.BucketCache(843): removed key 2307686003434224477_0 from back map with offset lock 4193280 in the evict process 2025-01-10T07:53:44,770 INFO [Thread-18 {}] bucket.BucketCache(1625): Finished validating 1 keys in the backing map. Recovered: 0. This took 1ms. 2025-01-10T07:53:44,829 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2025-01-10T07:53:44,929 INFO [Time-limited test {}] bucket.BucketCache(1726): Disabling cache 2025-01-10T07:53:44,930 INFO [Time-limited test {}] bucket.BucketCache(1750): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d6bd01cd-a623-1d16-5ab4-57fa19bd0211/bucket.persistence 2025-01-10T07:53:44,930 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2025-01-10T07:53:44,930 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2025-01-10T07:53:44,930 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2025-01-10T07:53:44,930 INFO [Time-limited test {}] bucket.BucketCache(1756): Shutting down cache persister thread. 2025-01-10T07:53:44,930 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2025-01-10T07:53:44,940 DEBUG [Time-limited test {}] bucket.BucketCache(1474): Thread Time-limited test started persisting bucket cache to file 2025-01-10T07:53:44,940 DEBUG [Time-limited test {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:44,941 DEBUG [Time-limited test {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:44,944 DEBUG [Time-limited test {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2025-01-10T07:53:44,944 DEBUG [Time-limited test {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1736495624940 2025-01-10T07:53:44,944 DEBUG [Time-limited test {}] bucket.BucketCache(1496): Thread Time-limited test finished persisting bucket cache to file, renaming 2025-01-10T07:53:44,958 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d6bd01cd-a623-1d16-5ab4-57fa19bd0211/bucket.cache 2025-01-10T07:53:44,958 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:44,958 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:44,958 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2025-01-10T07:53:44,959 INFO [Thread-20 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:44,959 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d6bd01cd-a623-1d16-5ab4-57fa19bd0211/bucket.cache, capacity=32 MB, blockSize=8 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d6bd01cd-a623-1d16-5ab4-57fa19bd0211/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:44,959 WARN [Thread-20 {}] bucket.BucketCache(1520): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2025-01-10T07:53:44,959 INFO [Thread-20 {}] bucket.BucketAllocator(343): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:44,959 INFO [Thread-20 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d6bd01cd-a623-1d16-5ab4-57fa19bd0211/bucket.persistence is complete. 2025-01-10T07:53:45,065 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestVerifyBucketCacheFile#testRetrieveFromFile[0: blockSize=8,192, bucketSizes=null] Thread=28 (was 23) - Thread LEAK? -, OpenFileDescriptor=289 (was 288) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=558 (was 558), ProcessCount=11 (was 11), AvailableMemoryMB=6594 (was 6600) 2025-01-10T07:53:45,071 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestVerifyBucketCacheFile#testModifiedBucketCacheFileData[0: blockSize=8,192, bucketSizes=null] Thread=28, OpenFileDescriptor=289, MaxFileDescriptor=1048576, SystemLoadAverage=558, ProcessCount=11, AvailableMemoryMB=6594 2025-01-10T07:53:45,085 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-log-dir so I do NOT create it in target/test-data/7710cb5e-4abc-151f-5509-17a961013923 2025-01-10T07:53:45,085 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.log.dir property value differs in configuration and system: Configuration=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/../logs while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-log-dir Erasing configuration value by system value. 2025-01-10T07:53:45,086 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-tmp-dir so I do NOT create it in target/test-data/7710cb5e-4abc-151f-5509-17a961013923 2025-01-10T07:53:45,086 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-tmp-dir Erasing configuration value by system value. 2025-01-10T07:53:45,086 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7710cb5e-4abc-151f-5509-17a961013923 2025-01-10T07:53:45,087 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-10T07:53:45,106 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7710cb5e-4abc-151f-5509-17a961013923/bucket.cache 2025-01-10T07:53:45,106 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:45,106 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:45,106 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 9223372036854775807 2025-01-10T07:53:45,107 INFO [Thread-22 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:45,107 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7710cb5e-4abc-151f-5509-17a961013923/bucket.cache, capacity=32 MB, blockSize=8 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7710cb5e-4abc-151f-5509-17a961013923/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:45,107 WARN [Thread-22 {}] bucket.BucketCache(1520): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2025-01-10T07:53:45,107 INFO [Thread-22 {}] bucket.BucketAllocator(343): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:45,107 INFO [Thread-22 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7710cb5e-4abc-151f-5509-17a961013923/bucket.persistence is complete. 2025-01-10T07:53:45,308 INFO [Time-limited test {}] bucket.BucketCache(1726): Disabling cache 2025-01-10T07:53:45,308 INFO [Time-limited test {}] bucket.BucketCache(1750): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7710cb5e-4abc-151f-5509-17a961013923/bucket.persistence 2025-01-10T07:53:45,308 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2025-01-10T07:53:45,308 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2025-01-10T07:53:45,308 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2025-01-10T07:53:45,308 INFO [Time-limited test {}] bucket.BucketCache(1756): Shutting down cache persister thread. 2025-01-10T07:53:45,309 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2025-01-10T07:53:45,319 DEBUG [Time-limited test {}] bucket.BucketCache(1474): Thread Time-limited test started persisting bucket cache to file 2025-01-10T07:53:45,319 DEBUG [Time-limited test {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:45,319 DEBUG [Time-limited test {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:45,323 DEBUG [Time-limited test {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2025-01-10T07:53:45,323 DEBUG [Time-limited test {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1736495625319 2025-01-10T07:53:45,323 DEBUG [Time-limited test {}] bucket.BucketCache(1496): Thread Time-limited test finished persisting bucket cache to file, renaming 2025-01-10T07:53:45,339 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7710cb5e-4abc-151f-5509-17a961013923/bucket.cache 2025-01-10T07:53:45,339 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:45,339 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:45,339 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2025-01-10T07:53:45,340 INFO [Thread-24 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:45,340 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7710cb5e-4abc-151f-5509-17a961013923/bucket.cache, capacity=32 MB, blockSize=8 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7710cb5e-4abc-151f-5509-17a961013923/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:45,340 INFO [Thread-24 {}] bucket.BucketCache(1540): Reading new chunked format of persistence. 2025-01-10T07:53:45,341 INFO [Thread-24 {}] bucket.BucketCache(1693): Retrieved 1 of chunks with blockCount = 1. 2025-01-10T07:53:45,344 WARN [Thread-24 {}] bucket.BucketCache(1601): Checksum for cache file failed. We need to validate each cache key in the backing map. This may take some time, so we'll do it in a background thread, 2025-01-10T07:53:45,344 DEBUG [Thread-24 {}] bucket.BucketCache(1557): Updating region size map after retrieving cached file list 2025-01-10T07:53:45,345 INFO [Thread-24 {}] bucket.BucketAllocator(343): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:45,345 INFO [Thread-24 {}] bucket.BucketCache(1550): Bucket cache retrieved from file successfully with size: 1 2025-01-10T07:53:45,345 INFO [Thread-24 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7710cb5e-4abc-151f-5509-17a961013923/bucket.persistence is complete. 2025-01-10T07:53:45,395 DEBUG [Thread-26 {}] bucket.BucketCache(1619): Check for key -8273215103298453870_0 failed. Evicting. 2025-01-10T07:53:45,395 DEBUG [Thread-26 {}] bucket.BucketCache(843): removed key -8273215103298453870_0 from back map with offset lock 4193280 in the evict process 2025-01-10T07:53:45,395 INFO [Thread-26 {}] bucket.BucketCache(1625): Finished validating 1 keys in the backing map. Recovered: 0. This took 1ms. 2025-01-10T07:53:45,448 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestVerifyBucketCacheFile#testModifiedBucketCacheFileData[0: blockSize=8,192, bucketSizes=null] Thread=33 (was 28) - Thread LEAK? -, OpenFileDescriptor=290 (was 289) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=530 (was 558), ProcessCount=11 (was 11), AvailableMemoryMB=6592 (was 6594) 2025-01-10T07:53:45,456 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestVerifyBucketCacheFile#testModifiedBucketCacheFileTime[0: blockSize=8,192, bucketSizes=null] Thread=33, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=530, ProcessCount=11, AvailableMemoryMB=6592 2025-01-10T07:53:45,471 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-log-dir so I do NOT create it in target/test-data/088e174c-7fea-2eeb-ebfa-5834b72fd3db 2025-01-10T07:53:45,471 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.log.dir property value differs in configuration and system: Configuration=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/../logs while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-log-dir Erasing configuration value by system value. 2025-01-10T07:53:45,471 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-tmp-dir so I do NOT create it in target/test-data/088e174c-7fea-2eeb-ebfa-5834b72fd3db 2025-01-10T07:53:45,471 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-tmp-dir Erasing configuration value by system value. 2025-01-10T07:53:45,472 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/088e174c-7fea-2eeb-ebfa-5834b72fd3db 2025-01-10T07:53:45,472 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-10T07:53:45,490 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/088e174c-7fea-2eeb-ebfa-5834b72fd3db/bucket.cache 2025-01-10T07:53:45,490 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:45,490 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:45,490 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2025-01-10T07:53:45,491 INFO [Thread-28 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:45,491 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/088e174c-7fea-2eeb-ebfa-5834b72fd3db/bucket.cache, capacity=32 MB, blockSize=8 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/088e174c-7fea-2eeb-ebfa-5834b72fd3db/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:45,491 WARN [Thread-28 {}] bucket.BucketCache(1520): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2025-01-10T07:53:45,491 INFO [Thread-28 {}] bucket.BucketAllocator(343): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:45,491 INFO [Thread-28 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/088e174c-7fea-2eeb-ebfa-5834b72fd3db/bucket.persistence is complete. 2025-01-10T07:53:45,691 INFO [Time-limited test {}] bucket.BucketCache(1726): Disabling cache 2025-01-10T07:53:45,692 INFO [Time-limited test {}] bucket.BucketCache(1750): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/088e174c-7fea-2eeb-ebfa-5834b72fd3db/bucket.persistence 2025-01-10T07:53:45,692 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2025-01-10T07:53:45,692 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2025-01-10T07:53:45,692 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2025-01-10T07:53:45,692 INFO [Time-limited test {}] bucket.BucketCache(1756): Shutting down cache persister thread. 2025-01-10T07:53:45,693 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2025-01-10T07:53:45,703 DEBUG [Time-limited test {}] bucket.BucketCache(1474): Thread Time-limited test started persisting bucket cache to file 2025-01-10T07:53:45,703 DEBUG [Time-limited test {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:45,703 DEBUG [Time-limited test {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:45,707 DEBUG [Time-limited test {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2025-01-10T07:53:45,707 DEBUG [Time-limited test {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1736495625703 2025-01-10T07:53:45,707 DEBUG [Time-limited test {}] bucket.BucketCache(1496): Thread Time-limited test finished persisting bucket cache to file, renaming 2025-01-10T07:53:45,723 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/088e174c-7fea-2eeb-ebfa-5834b72fd3db/bucket.cache 2025-01-10T07:53:45,723 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:45,723 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:45,724 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2025-01-10T07:53:45,724 INFO [Thread-30 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:45,724 INFO [Thread-30 {}] bucket.BucketCache(1540): Reading new chunked format of persistence. 2025-01-10T07:53:45,724 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/088e174c-7fea-2eeb-ebfa-5834b72fd3db/bucket.cache, capacity=32 MB, blockSize=8 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/088e174c-7fea-2eeb-ebfa-5834b72fd3db/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:45,725 INFO [Thread-30 {}] bucket.BucketCache(1693): Retrieved 1 of chunks with blockCount = 1. 2025-01-10T07:53:45,728 WARN [Thread-30 {}] bucket.BucketCache(1601): Checksum for cache file failed. We need to validate each cache key in the backing map. This may take some time, so we'll do it in a background thread, 2025-01-10T07:53:45,728 DEBUG [Thread-30 {}] bucket.BucketCache(1557): Updating region size map after retrieving cached file list 2025-01-10T07:53:45,728 INFO [Thread-30 {}] bucket.BucketAllocator(343): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:45,728 INFO [Thread-30 {}] bucket.BucketCache(1550): Bucket cache retrieved from file successfully with size: 1 2025-01-10T07:53:45,729 INFO [Thread-30 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/088e174c-7fea-2eeb-ebfa-5834b72fd3db/bucket.persistence is complete. 2025-01-10T07:53:45,778 INFO [Thread-32 {}] bucket.BucketCache(1625): Finished validating 1 keys in the backing map. Recovered: 1. This took 0ms. 2025-01-10T07:53:45,831 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestVerifyBucketCacheFile#testModifiedBucketCacheFileTime[0: blockSize=8,192, bucketSizes=null] Thread=38 (was 33) - Thread LEAK? -, OpenFileDescriptor=291 (was 290) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=530 (was 530), ProcessCount=11 (was 11), AvailableMemoryMB=6586 (was 6592) 2025-01-10T07:53:45,837 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestVerifyBucketCacheFile#testRetrieveFromFileAfterDelete[0: blockSize=8,192, bucketSizes=null] Thread=38, OpenFileDescriptor=291, MaxFileDescriptor=1048576, SystemLoadAverage=530, ProcessCount=11, AvailableMemoryMB=6586 2025-01-10T07:53:45,853 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-log-dir so I do NOT create it in target/test-data/b2d214c2-453c-997b-a175-27421733393a 2025-01-10T07:53:45,853 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.log.dir property value differs in configuration and system: Configuration=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/../logs while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-log-dir Erasing configuration value by system value. 2025-01-10T07:53:45,853 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-tmp-dir so I do NOT create it in target/test-data/b2d214c2-453c-997b-a175-27421733393a 2025-01-10T07:53:45,853 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-tmp-dir Erasing configuration value by system value. 2025-01-10T07:53:45,854 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b2d214c2-453c-997b-a175-27421733393a 2025-01-10T07:53:45,854 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-10T07:53:45,857 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b2d214c2-453c-997b-a175-27421733393a/bucket.cache 2025-01-10T07:53:45,858 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:45,858 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:45,858 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 300 2025-01-10T07:53:45,859 INFO [Thread-34 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:45,859 WARN [Thread-34 {}] bucket.BucketCache(1520): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2025-01-10T07:53:45,859 INFO [Thread-34 {}] bucket.BucketAllocator(343): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:45,859 INFO [Thread-34 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b2d214c2-453c-997b-a175-27421733393a/bucket.persistence1736495625857 is complete. 2025-01-10T07:53:45,862 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b2d214c2-453c-997b-a175-27421733393a/bucket.cache, capacity=32 MB, blockSize=8 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b2d214c2-453c-997b-a175-27421733393a/bucket.persistence1736495625857, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:45,959 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:45,959 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:45,959 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d6bd01cd-a623-1d16-5ab4-57fa19bd0211/bucket.persistence1736495625959 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:45,959 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d6bd01cd-a623-1d16-5ab4-57fa19bd0211/bucket.persistence1736495625959 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:45,962 INFO [Time-limited test {}] bucket.BucketCache(1726): Disabling cache 2025-01-10T07:53:45,962 INFO [Time-limited test {}] bucket.BucketCache(1750): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b2d214c2-453c-997b-a175-27421733393a/bucket.persistence1736495625857 2025-01-10T07:53:45,962 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2025-01-10T07:53:45,962 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2025-01-10T07:53:45,962 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2025-01-10T07:53:45,963 INFO [Time-limited test {}] bucket.BucketCache(1756): Shutting down cache persister thread. 2025-01-10T07:53:45,963 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2025-01-10T07:53:45,973 DEBUG [Time-limited test {}] bucket.BucketCache(1474): Thread Time-limited test started persisting bucket cache to file 2025-01-10T07:53:45,973 DEBUG [Time-limited test {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:45,973 DEBUG [Time-limited test {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:45,977 DEBUG [Time-limited test {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2025-01-10T07:53:45,977 DEBUG [Time-limited test {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence17364956258571736495625973 2025-01-10T07:53:45,977 DEBUG [Time-limited test {}] bucket.BucketCache(1496): Thread Time-limited test finished persisting bucket cache to file, renaming 2025-01-10T07:53:46,328 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b2d214c2-453c-997b-a175-27421733393a/bucket.cache 2025-01-10T07:53:46,328 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:46,329 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:46,329 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 300 2025-01-10T07:53:46,329 INFO [Thread-36 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:46,329 WARN [Thread-36 {}] bucket.BucketCache(1520): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2025-01-10T07:53:46,330 INFO [Thread-36 {}] bucket.BucketAllocator(343): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:46,330 INFO [Thread-36 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b2d214c2-453c-997b-a175-27421733393a/bucket.persistence1736495625857 is complete. 2025-01-10T07:53:46,331 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b2d214c2-453c-997b-a175-27421733393a/bucket.cache, capacity=32 MB, blockSize=8 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b2d214c2-453c-997b-a175-27421733393a/bucket.persistence1736495625857, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:46,338 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestVerifyBucketCacheFile#testRetrieveFromFileAfterDelete[0: blockSize=8,192, bucketSizes=null] Thread=42 (was 38) Potentially hanging thread: Thread-36 java.base@17.0.11/java.lang.Thread.start0(Native Method) java.base@17.0.11/java.lang.Thread.start(Thread.java:809) app//org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.startWriterThreads(BucketCache.java:460) app//org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.lambda$startPersistenceRetriever$0(BucketCache.java:415) app//org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$$Lambda$163/0x00007f457420e910.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=292 (was 291) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=530 (was 530), ProcessCount=11 (was 11), AvailableMemoryMB=6582 (was 6586) 2025-01-10T07:53:46,340 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:46,340 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:46,340 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7710cb5e-4abc-151f-5509-17a961013923/bucket.persistence1736495626340 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:46,340 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7710cb5e-4abc-151f-5509-17a961013923/bucket.persistence1736495626340 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:46,344 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestVerifyBucketCacheFile#testPartiallyFilledChunks[0: blockSize=8,192, bucketSizes=null] Thread=43, OpenFileDescriptor=292, MaxFileDescriptor=1048576, SystemLoadAverage=530, ProcessCount=11, AvailableMemoryMB=6582 2025-01-10T07:53:46,359 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-log-dir so I do NOT create it in target/test-data/81823344-9ae7-f60d-9761-52816b12ba6b 2025-01-10T07:53:46,359 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.log.dir property value differs in configuration and system: Configuration=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/../logs while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-log-dir Erasing configuration value by system value. 2025-01-10T07:53:46,360 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-tmp-dir so I do NOT create it in target/test-data/81823344-9ae7-f60d-9761-52816b12ba6b 2025-01-10T07:53:46,360 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-tmp-dir Erasing configuration value by system value. 2025-01-10T07:53:46,360 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/81823344-9ae7-f60d-9761-52816b12ba6b 2025-01-10T07:53:46,361 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-10T07:53:46,381 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/81823344-9ae7-f60d-9761-52816b12ba6b/bucket.cache 2025-01-10T07:53:46,381 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:46,382 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:46,382 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2025-01-10T07:53:46,385 INFO [Thread-38 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:46,385 WARN [Thread-38 {}] bucket.BucketCache(1520): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2025-01-10T07:53:46,385 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/81823344-9ae7-f60d-9761-52816b12ba6b/bucket.cache, capacity=32 MB, blockSize=8 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/81823344-9ae7-f60d-9761-52816b12ba6b/bucket.persistence1736495626381, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:46,385 INFO [Thread-38 {}] bucket.BucketAllocator(343): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:46,386 INFO [Thread-38 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/81823344-9ae7-f60d-9761-52816b12ba6b/bucket.persistence1736495626381 is complete. 2025-01-10T07:53:46,629 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:46,629 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:46,630 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:46,630 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:46,633 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:46,633 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956258571736495626629 2025-01-10T07:53:46,633 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:46,724 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:46,724 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:46,724 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/088e174c-7fea-2eeb-ebfa-5834b72fd3db/bucket.persistence1736495626724 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:46,724 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/088e174c-7fea-2eeb-ebfa-5834b72fd3db/bucket.persistence1736495626724 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:46,934 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:46,934 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:46,934 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:46,934 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:46,937 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:46,938 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956258571736495626934 2025-01-10T07:53:46,938 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:47,238 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:47,238 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:47,239 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:47,239 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:47,242 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:47,242 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956258571736495627238 2025-01-10T07:53:47,242 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:47,341 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:47,341 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:47,341 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7710cb5e-4abc-151f-5509-17a961013923/bucket.persistence1736495627341 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:47,341 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7710cb5e-4abc-151f-5509-17a961013923/bucket.persistence1736495627341 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:47,382 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:47,382 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:47,382 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:47,383 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 9, fullycachedFiles size: 0, chunkSize: 5 2025-01-10T07:53:47,387 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 9, fullycachedFiles size: 0 2025-01-10T07:53:47,387 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 9, fullycachedFiles size: 0, file name: bucket.persistence17364956263811736495627382 2025-01-10T07:53:47,387 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:47,543 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:47,543 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:47,543 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:47,543 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:47,546 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:47,547 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956258571736495627543 2025-01-10T07:53:47,547 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:47,725 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:47,725 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:47,725 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/088e174c-7fea-2eeb-ebfa-5834b72fd3db/bucket.persistence1736495627725 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:47,725 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/088e174c-7fea-2eeb-ebfa-5834b72fd3db/bucket.persistence1736495627725 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:47,788 DEBUG [Time-limited test {}] bucket.BucketCache(1474): Thread Time-limited test started persisting bucket cache to file 2025-01-10T07:53:47,789 DEBUG [Time-limited test {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:47,789 DEBUG [Time-limited test {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 13, fullycachedFiles size: 0, chunkSize: 5 2025-01-10T07:53:47,793 DEBUG [Time-limited test {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 13, fullycachedFiles size: 0 2025-01-10T07:53:47,793 DEBUG [Time-limited test {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 13, fullycachedFiles size: 0, file name: bucket.persistence17364956263811736495627789 2025-01-10T07:53:47,793 DEBUG [Time-limited test {}] bucket.BucketCache(1496): Thread Time-limited test finished persisting bucket cache to file, renaming 2025-01-10T07:53:47,793 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/81823344-9ae7-f60d-9761-52816b12ba6b/bucket.cache 2025-01-10T07:53:47,794 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:47,794 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:47,794 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2025-01-10T07:53:47,795 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/81823344-9ae7-f60d-9761-52816b12ba6b/bucket.cache, capacity=32 MB, blockSize=8 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/81823344-9ae7-f60d-9761-52816b12ba6b/bucket.persistence1736495626381, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:47,795 INFO [Thread-45 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:47,795 INFO [Thread-45 {}] bucket.BucketCache(1540): Reading new chunked format of persistence. 2025-01-10T07:53:47,796 INFO [Thread-45 {}] bucket.BucketCache(1693): Retrieved 3 of chunks with blockCount = 13. 2025-01-10T07:53:47,800 DEBUG [Thread-45 {}] bucket.BucketCache(1557): Updating region size map after retrieving cached file list 2025-01-10T07:53:47,800 INFO [Thread-45 {}] bucket.BucketAllocator(343): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:47,800 INFO [Thread-45 {}] bucket.BucketCache(1550): Bucket cache retrieved from file successfully with size: 13 2025-01-10T07:53:47,800 INFO [Thread-45 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/81823344-9ae7-f60d-9761-52816b12ba6b/bucket.persistence1736495626381 is complete. 2025-01-10T07:53:47,847 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:47,847 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:47,847 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:47,847 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:47,851 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:47,851 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956258571736495627847 2025-01-10T07:53:47,851 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:47,904 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestVerifyBucketCacheFile#testPartiallyFilledChunks[0: blockSize=8,192, bucketSizes=null] Thread=53 (was 43) - Thread LEAK? -, OpenFileDescriptor=294 (was 292) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=530 (was 530), ProcessCount=11 (was 11), AvailableMemoryMB=6583 (was 6582) - AvailableMemoryMB LEAK? - 2025-01-10T07:53:47,910 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestVerifyBucketCacheFile#testCompletelyFilledChunks[0: blockSize=8,192, bucketSizes=null] Thread=53, OpenFileDescriptor=294, MaxFileDescriptor=1048576, SystemLoadAverage=530, ProcessCount=11, AvailableMemoryMB=6581 2025-01-10T07:53:47,925 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-log-dir so I do NOT create it in target/test-data/8d68ec26-bb55-e9b9-9e09-997e8dcb41f6 2025-01-10T07:53:47,925 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.log.dir property value differs in configuration and system: Configuration=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/../logs while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-log-dir Erasing configuration value by system value. 2025-01-10T07:53:47,925 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-tmp-dir so I do NOT create it in target/test-data/8d68ec26-bb55-e9b9-9e09-997e8dcb41f6 2025-01-10T07:53:47,925 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-tmp-dir Erasing configuration value by system value. 2025-01-10T07:53:47,926 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8d68ec26-bb55-e9b9-9e09-997e8dcb41f6 2025-01-10T07:53:47,926 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-10T07:53:47,944 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8d68ec26-bb55-e9b9-9e09-997e8dcb41f6/bucket.cache 2025-01-10T07:53:47,944 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:47,944 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:47,944 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2025-01-10T07:53:47,945 INFO [Thread-49 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:47,945 WARN [Thread-49 {}] bucket.BucketCache(1520): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2025-01-10T07:53:47,945 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8d68ec26-bb55-e9b9-9e09-997e8dcb41f6/bucket.cache, capacity=32 MB, blockSize=8 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8d68ec26-bb55-e9b9-9e09-997e8dcb41f6/bucket.persistence1736495627944, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:47,945 INFO [Thread-49 {}] bucket.BucketAllocator(343): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:47,945 INFO [Thread-49 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8d68ec26-bb55-e9b9-9e09-997e8dcb41f6/bucket.persistence1736495627944 is complete. 2025-01-10T07:53:48,151 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:48,151 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:48,152 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:48,152 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:48,155 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:48,155 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956258571736495628151 2025-01-10T07:53:48,155 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:48,342 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:48,342 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:48,342 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7710cb5e-4abc-151f-5509-17a961013923/bucket.persistence1736495628342 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:48,342 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7710cb5e-4abc-151f-5509-17a961013923/bucket.persistence1736495628342 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:48,387 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:48,388 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:48,388 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/81823344-9ae7-f60d-9761-52816b12ba6b/bucket.persistence17364956263811736495628388 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:48,388 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/81823344-9ae7-f60d-9761-52816b12ba6b/bucket.persistence17364956263811736495628388 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:48,455 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:48,456 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:48,456 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:48,456 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:48,460 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:48,460 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956258571736495628456 2025-01-10T07:53:48,460 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:48,726 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:48,726 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:48,726 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/088e174c-7fea-2eeb-ebfa-5834b72fd3db/bucket.persistence1736495628726 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:48,726 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/088e174c-7fea-2eeb-ebfa-5834b72fd3db/bucket.persistence1736495628726 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:48,760 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:48,760 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:48,760 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:48,761 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:48,765 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:48,765 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956258571736495628760 2025-01-10T07:53:48,765 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:48,945 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:48,945 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:48,945 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:48,945 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 9, fullycachedFiles size: 0, chunkSize: 5 2025-01-10T07:53:48,950 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 9, fullycachedFiles size: 0 2025-01-10T07:53:48,950 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 9, fullycachedFiles size: 0, file name: bucket.persistence17364956279441736495628945 2025-01-10T07:53:48,950 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:49,048 DEBUG [Time-limited test {}] bucket.BucketCache(1474): Thread Time-limited test started persisting bucket cache to file 2025-01-10T07:53:49,048 DEBUG [Time-limited test {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:49,048 DEBUG [Time-limited test {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 10, fullycachedFiles size: 0, chunkSize: 5 2025-01-10T07:53:49,053 DEBUG [Time-limited test {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 10, fullycachedFiles size: 0 2025-01-10T07:53:49,053 DEBUG [Time-limited test {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 10, fullycachedFiles size: 0, file name: bucket.persistence17364956279441736495629048 2025-01-10T07:53:49,053 DEBUG [Time-limited test {}] bucket.BucketCache(1496): Thread Time-limited test finished persisting bucket cache to file, renaming 2025-01-10T07:53:49,053 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8d68ec26-bb55-e9b9-9e09-997e8dcb41f6/bucket.cache 2025-01-10T07:53:49,054 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:49,054 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:49,054 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2025-01-10T07:53:49,055 INFO [Thread-55 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:49,055 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8d68ec26-bb55-e9b9-9e09-997e8dcb41f6/bucket.cache, capacity=32 MB, blockSize=8 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8d68ec26-bb55-e9b9-9e09-997e8dcb41f6/bucket.persistence1736495627944, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:49,055 INFO [Thread-55 {}] bucket.BucketCache(1540): Reading new chunked format of persistence. 2025-01-10T07:53:49,056 INFO [Thread-55 {}] bucket.BucketCache(1693): Retrieved 2 of chunks with blockCount = 10. 2025-01-10T07:53:49,059 DEBUG [Thread-55 {}] bucket.BucketCache(1557): Updating region size map after retrieving cached file list 2025-01-10T07:53:49,059 INFO [Thread-55 {}] bucket.BucketAllocator(343): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:49,059 INFO [Thread-55 {}] bucket.BucketCache(1550): Bucket cache retrieved from file successfully with size: 10 2025-01-10T07:53:49,059 INFO [Thread-55 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8d68ec26-bb55-e9b9-9e09-997e8dcb41f6/bucket.persistence1736495627944 is complete. 2025-01-10T07:53:49,164 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestVerifyBucketCacheFile#testCompletelyFilledChunks[0: blockSize=8,192, bucketSizes=null] Thread=63 (was 53) - Thread LEAK? -, OpenFileDescriptor=296 (was 294) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=530 (was 530), ProcessCount=11 (was 11), AvailableMemoryMB=6485 (was 6581) 2025-01-10T07:53:49,171 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestVerifyBucketCacheFile#testSingleChunk[0: blockSize=8,192, bucketSizes=null] Thread=63, OpenFileDescriptor=296, MaxFileDescriptor=1048576, SystemLoadAverage=530, ProcessCount=11, AvailableMemoryMB=6485 2025-01-10T07:53:49,186 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-log-dir so I do NOT create it in target/test-data/d7fb7f24-0a10-4128-c40c-2ca16622cf4e 2025-01-10T07:53:49,187 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.log.dir property value differs in configuration and system: Configuration=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/../logs while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-log-dir Erasing configuration value by system value. 2025-01-10T07:53:49,187 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-tmp-dir so I do NOT create it in target/test-data/d7fb7f24-0a10-4128-c40c-2ca16622cf4e 2025-01-10T07:53:49,187 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-tmp-dir Erasing configuration value by system value. 2025-01-10T07:53:49,187 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d7fb7f24-0a10-4128-c40c-2ca16622cf4e 2025-01-10T07:53:49,188 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-10T07:53:49,206 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d7fb7f24-0a10-4128-c40c-2ca16622cf4e/bucket.cache 2025-01-10T07:53:49,206 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:49,207 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:49,207 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2025-01-10T07:53:49,207 INFO [Thread-58 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:49,207 WARN [Thread-58 {}] bucket.BucketCache(1520): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2025-01-10T07:53:49,208 INFO [Thread-58 {}] bucket.BucketAllocator(343): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:49,208 INFO [Thread-58 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d7fb7f24-0a10-4128-c40c-2ca16622cf4e/bucket.persistence1736495629206 is complete. 2025-01-10T07:53:49,208 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d7fb7f24-0a10-4128-c40c-2ca16622cf4e/bucket.cache, capacity=32 MB, blockSize=8 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d7fb7f24-0a10-4128-c40c-2ca16622cf4e/bucket.persistence1736495629206, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:49,343 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:49,343 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:49,343 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7710cb5e-4abc-151f-5509-17a961013923/bucket.persistence1736495629343 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:49,343 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7710cb5e-4abc-151f-5509-17a961013923/bucket.persistence1736495629343 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:49,366 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:49,366 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:49,366 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:49,366 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:49,370 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:49,370 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956258571736495629366 2025-01-10T07:53:49,370 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:49,671 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:49,671 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:49,671 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:49,671 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:49,675 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:49,675 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956258571736495629671 2025-01-10T07:53:49,675 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:49,710 DEBUG [Time-limited test {}] bucket.BucketCache(1474): Thread Time-limited test started persisting bucket cache to file 2025-01-10T07:53:49,710 DEBUG [Time-limited test {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:49,710 DEBUG [Time-limited test {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 5, fullycachedFiles size: 0, chunkSize: 5 2025-01-10T07:53:49,716 DEBUG [Time-limited test {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 5, fullycachedFiles size: 0 2025-01-10T07:53:49,717 DEBUG [Time-limited test {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 5, fullycachedFiles size: 0, file name: bucket.persistence17364956292061736495629710 2025-01-10T07:53:49,717 DEBUG [Time-limited test {}] bucket.BucketCache(1496): Thread Time-limited test finished persisting bucket cache to file, renaming 2025-01-10T07:53:49,717 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d7fb7f24-0a10-4128-c40c-2ca16622cf4e/bucket.cache 2025-01-10T07:53:49,717 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:49,718 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:49,718 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2025-01-10T07:53:49,718 INFO [Thread-62 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:49,718 INFO [Thread-62 {}] bucket.BucketCache(1540): Reading new chunked format of persistence. 2025-01-10T07:53:49,719 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d7fb7f24-0a10-4128-c40c-2ca16622cf4e/bucket.cache, capacity=32 MB, blockSize=8 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d7fb7f24-0a10-4128-c40c-2ca16622cf4e/bucket.persistence1736495629206, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:49,719 INFO [Thread-62 {}] bucket.BucketCache(1693): Retrieved 1 of chunks with blockCount = 5. 2025-01-10T07:53:49,725 DEBUG [Thread-62 {}] bucket.BucketCache(1557): Updating region size map after retrieving cached file list 2025-01-10T07:53:49,726 INFO [Thread-62 {}] bucket.BucketAllocator(343): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:49,726 INFO [Thread-62 {}] bucket.BucketCache(1550): Bucket cache retrieved from file successfully with size: 5 2025-01-10T07:53:49,726 INFO [Thread-62 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d7fb7f24-0a10-4128-c40c-2ca16622cf4e/bucket.persistence1736495629206 is complete. 2025-01-10T07:53:49,831 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestVerifyBucketCacheFile#testSingleChunk[0: blockSize=8,192, bucketSizes=null] Thread=73 (was 63) - Thread LEAK? -, OpenFileDescriptor=298 (was 296) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=530 (was 530), ProcessCount=11 (was 11), AvailableMemoryMB=6350 (was 6485) 2025-01-10T07:53:49,839 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestVerifyBucketCacheFile#testBucketCacheRecovery[1: blockSize=16,384, bucketSizes=[I@388526fb] Thread=73, OpenFileDescriptor=298, MaxFileDescriptor=1048576, SystemLoadAverage=530, ProcessCount=11, AvailableMemoryMB=6348 2025-01-10T07:53:49,862 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-log-dir so I do NOT create it in target/test-data/fce6151a-bc6f-23ae-890d-82f9a2f6ba09 2025-01-10T07:53:49,862 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.log.dir property value differs in configuration and system: Configuration=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/../logs while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-log-dir Erasing configuration value by system value. 2025-01-10T07:53:49,862 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-tmp-dir so I do NOT create it in target/test-data/fce6151a-bc6f-23ae-890d-82f9a2f6ba09 2025-01-10T07:53:49,862 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-tmp-dir Erasing configuration value by system value. 2025-01-10T07:53:49,862 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/fce6151a-bc6f-23ae-890d-82f9a2f6ba09 2025-01-10T07:53:49,863 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-10T07:53:49,889 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/fce6151a-bc6f-23ae-890d-82f9a2f6ba09/bucket.cache 2025-01-10T07:53:49,890 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:49,890 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:49,890 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 9223372036854775807 2025-01-10T07:53:49,892 INFO [Thread-65 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:49,893 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/fce6151a-bc6f-23ae-890d-82f9a2f6ba09/bucket.cache, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/fce6151a-bc6f-23ae-890d-82f9a2f6ba09/bucket.persistence1736495629889, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:49,893 WARN [Thread-65 {}] bucket.BucketCache(1520): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2025-01-10T07:53:49,893 INFO [Thread-65 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:49,893 INFO [Thread-65 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/fce6151a-bc6f-23ae-890d-82f9a2f6ba09/bucket.persistence1736495629889 is complete. 2025-01-10T07:53:50,054 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:50,054 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:50,055 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8d68ec26-bb55-e9b9-9e09-997e8dcb41f6/bucket.persistence17364956279441736495630055 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:50,055 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8d68ec26-bb55-e9b9-9e09-997e8dcb41f6/bucket.persistence17364956279441736495630055 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:50,207 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:50,208 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:50,208 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d7fb7f24-0a10-4128-c40c-2ca16622cf4e/bucket.persistence17364956292061736495630208 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:50,208 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d7fb7f24-0a10-4128-c40c-2ca16622cf4e/bucket.persistence17364956292061736495630208 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:50,276 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:50,276 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:50,276 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:50,277 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:50,281 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:50,281 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956258571736495630276 2025-01-10T07:53:50,281 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:50,294 DEBUG [Time-limited test {}] bucket.BucketCache(1474): Thread Time-limited test started persisting bucket cache to file 2025-01-10T07:53:50,294 DEBUG [Time-limited test {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:50,294 DEBUG [Time-limited test {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 3, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:50,299 DEBUG [Time-limited test {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 3, fullycachedFiles size: 0 2025-01-10T07:53:50,299 DEBUG [Time-limited test {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 3, fullycachedFiles size: 0, file name: bucket.persistence17364956298891736495630294 2025-01-10T07:53:50,299 DEBUG [Time-limited test {}] bucket.BucketCache(1496): Thread Time-limited test finished persisting bucket cache to file, renaming 2025-01-10T07:53:50,299 DEBUG [Time-limited test {}] bucket.BucketCache(843): removed key -3962381980284110197_0 from back map with offset lock 2089984 in the evict process 2025-01-10T07:53:50,344 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:50,344 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:50,344 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7710cb5e-4abc-151f-5509-17a961013923/bucket.persistence1736495630344 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:50,345 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7710cb5e-4abc-151f-5509-17a961013923/bucket.persistence1736495630344 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:50,388 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:50,389 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:50,389 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/81823344-9ae7-f60d-9761-52816b12ba6b/bucket.persistence17364956263811736495630389 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:50,389 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/81823344-9ae7-f60d-9761-52816b12ba6b/bucket.persistence17364956263811736495630389 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:50,400 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/fce6151a-bc6f-23ae-890d-82f9a2f6ba09/bucket.cache 2025-01-10T07:53:50,400 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:50,401 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:50,401 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 9223372036854775807 2025-01-10T07:53:50,401 INFO [Thread-68 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:50,401 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/fce6151a-bc6f-23ae-890d-82f9a2f6ba09/bucket.cache, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/fce6151a-bc6f-23ae-890d-82f9a2f6ba09/bucket.persistence1736495629889, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:50,401 INFO [Thread-68 {}] bucket.BucketCache(1540): Reading new chunked format of persistence. 2025-01-10T07:53:50,402 INFO [Thread-68 {}] bucket.BucketCache(1693): Retrieved 1 of chunks with blockCount = 3. 2025-01-10T07:53:50,406 WARN [Thread-68 {}] bucket.BucketCache(1601): Checksum for cache file failed. We need to validate each cache key in the backing map. This may take some time, so we'll do it in a background thread, 2025-01-10T07:53:50,406 DEBUG [Thread-68 {}] bucket.BucketCache(1557): Updating region size map after retrieving cached file list 2025-01-10T07:53:50,407 INFO [Thread-68 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:50,407 INFO [Thread-68 {}] bucket.BucketCache(1550): Bucket cache retrieved from file successfully with size: 3 2025-01-10T07:53:50,407 INFO [Thread-68 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/fce6151a-bc6f-23ae-890d-82f9a2f6ba09/bucket.persistence1736495629889 is complete. 2025-01-10T07:53:50,457 DEBUG [Thread-70 {}] bucket.BucketCache(1619): Check for key -3962381980284110197_0 failed. Evicting. 2025-01-10T07:53:50,457 DEBUG [Thread-70 {}] bucket.BucketCache(843): removed key -3962381980284110197_0 from back map with offset lock 2089984 in the evict process 2025-01-10T07:53:50,457 INFO [Thread-70 {}] bucket.BucketCache(1625): Finished validating 3 keys in the backing map. Recovered: 2. This took 0ms. 2025-01-10T07:53:50,511 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestVerifyBucketCacheFile#testBucketCacheRecovery[1: blockSize=16,384, bucketSizes=[I@388526fb] Thread=83 (was 73) - Thread LEAK? -, OpenFileDescriptor=300 (was 298) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=495 (was 530), ProcessCount=11 (was 11), AvailableMemoryMB=6240 (was 6348) 2025-01-10T07:53:50,521 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestVerifyBucketCacheFile#testRetrieveFromFile[1: blockSize=16,384, bucketSizes=[I@388526fb] Thread=83, OpenFileDescriptor=300, MaxFileDescriptor=1048576, SystemLoadAverage=495, ProcessCount=11, AvailableMemoryMB=6239 2025-01-10T07:53:50,542 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-log-dir so I do NOT create it in target/test-data/a3db39a4-a386-2af8-3c1e-c42d5c261030 2025-01-10T07:53:50,542 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.log.dir property value differs in configuration and system: Configuration=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/../logs while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-log-dir Erasing configuration value by system value. 2025-01-10T07:53:50,542 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-tmp-dir so I do NOT create it in target/test-data/a3db39a4-a386-2af8-3c1e-c42d5c261030 2025-01-10T07:53:50,542 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-tmp-dir Erasing configuration value by system value. 2025-01-10T07:53:50,542 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a3db39a4-a386-2af8-3c1e-c42d5c261030 2025-01-10T07:53:50,543 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-10T07:53:50,567 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a3db39a4-a386-2af8-3c1e-c42d5c261030/bucket.cache 2025-01-10T07:53:50,567 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:50,568 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:50,568 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2025-01-10T07:53:50,570 INFO [Thread-72 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:50,570 WARN [Thread-72 {}] bucket.BucketCache(1520): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2025-01-10T07:53:50,570 INFO [Thread-72 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:50,571 INFO [Thread-72 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a3db39a4-a386-2af8-3c1e-c42d5c261030/bucket.persistence is complete. 2025-01-10T07:53:50,571 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a3db39a4-a386-2af8-3c1e-c42d5c261030/bucket.cache, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a3db39a4-a386-2af8-3c1e-c42d5c261030/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:50,581 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:50,582 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:50,582 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:50,582 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:50,586 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:50,586 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956258571736495630582 2025-01-10T07:53:50,586 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:50,672 INFO [Time-limited test {}] bucket.BucketCache(1726): Disabling cache 2025-01-10T07:53:50,672 INFO [Time-limited test {}] bucket.BucketCache(1750): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a3db39a4-a386-2af8-3c1e-c42d5c261030/bucket.persistence 2025-01-10T07:53:50,672 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2025-01-10T07:53:50,672 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2025-01-10T07:53:50,672 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2025-01-10T07:53:50,672 INFO [Time-limited test {}] bucket.BucketCache(1756): Shutting down cache persister thread. 2025-01-10T07:53:50,673 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2025-01-10T07:53:50,683 DEBUG [Time-limited test {}] bucket.BucketCache(1474): Thread Time-limited test started persisting bucket cache to file 2025-01-10T07:53:50,683 DEBUG [Time-limited test {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:50,683 DEBUG [Time-limited test {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:50,688 DEBUG [Time-limited test {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2025-01-10T07:53:50,688 DEBUG [Time-limited test {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1736495630683 2025-01-10T07:53:50,688 DEBUG [Time-limited test {}] bucket.BucketCache(1496): Thread Time-limited test finished persisting bucket cache to file, renaming 2025-01-10T07:53:50,711 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a3db39a4-a386-2af8-3c1e-c42d5c261030/bucket.cache 2025-01-10T07:53:50,711 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:50,711 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:50,711 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2025-01-10T07:53:50,714 INFO [Thread-75 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:50,714 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a3db39a4-a386-2af8-3c1e-c42d5c261030/bucket.cache, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a3db39a4-a386-2af8-3c1e-c42d5c261030/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:50,714 INFO [Thread-75 {}] bucket.BucketCache(1540): Reading new chunked format of persistence. 2025-01-10T07:53:50,715 INFO [Thread-75 {}] bucket.BucketCache(1693): Retrieved 1 of chunks with blockCount = 1. 2025-01-10T07:53:50,722 DEBUG [Thread-75 {}] bucket.BucketCache(1557): Updating region size map after retrieving cached file list 2025-01-10T07:53:50,722 INFO [Thread-75 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:50,722 INFO [Thread-75 {}] bucket.BucketCache(1550): Bucket cache retrieved from file successfully with size: 1 2025-01-10T07:53:50,722 INFO [Thread-75 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a3db39a4-a386-2af8-3c1e-c42d5c261030/bucket.persistence is complete. 2025-01-10T07:53:50,814 INFO [Time-limited test {}] bucket.BucketCache(1726): Disabling cache 2025-01-10T07:53:50,815 INFO [Time-limited test {}] bucket.BucketCache(1750): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a3db39a4-a386-2af8-3c1e-c42d5c261030/bucket.persistence 2025-01-10T07:53:50,815 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2025-01-10T07:53:50,815 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2025-01-10T07:53:50,815 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2025-01-10T07:53:50,815 INFO [Time-limited test {}] bucket.BucketCache(1756): Shutting down cache persister thread. 2025-01-10T07:53:50,816 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2025-01-10T07:53:50,826 DEBUG [Time-limited test {}] bucket.BucketCache(1474): Thread Time-limited test started persisting bucket cache to file 2025-01-10T07:53:50,826 DEBUG [Time-limited test {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:50,826 DEBUG [Time-limited test {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:50,830 DEBUG [Time-limited test {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2025-01-10T07:53:50,831 DEBUG [Time-limited test {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1736495630826 2025-01-10T07:53:50,831 DEBUG [Time-limited test {}] bucket.BucketCache(1496): Thread Time-limited test finished persisting bucket cache to file, renaming 2025-01-10T07:53:50,854 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a3db39a4-a386-2af8-3c1e-c42d5c261030/bucket.cache 2025-01-10T07:53:50,854 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:50,854 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:50,854 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2025-01-10T07:53:50,855 INFO [Thread-78 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:50,855 INFO [Thread-78 {}] bucket.BucketCache(1540): Reading new chunked format of persistence. 2025-01-10T07:53:50,856 INFO [Thread-78 {}] bucket.BucketCache(1693): Retrieved 1 of chunks with blockCount = 1. 2025-01-10T07:53:50,857 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a3db39a4-a386-2af8-3c1e-c42d5c261030/bucket.cache, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a3db39a4-a386-2af8-3c1e-c42d5c261030/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:50,867 WARN [Thread-78 {}] bucket.BucketCache(1601): Checksum for cache file failed. We need to validate each cache key in the backing map. This may take some time, so we'll do it in a background thread, 2025-01-10T07:53:50,868 DEBUG [Thread-78 {}] bucket.BucketCache(1557): Updating region size map after retrieving cached file list 2025-01-10T07:53:50,868 INFO [Thread-78 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:50,868 INFO [Thread-78 {}] bucket.BucketCache(1550): Bucket cache retrieved from file successfully with size: 1 2025-01-10T07:53:50,868 INFO [Thread-78 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a3db39a4-a386-2af8-3c1e-c42d5c261030/bucket.persistence is complete. 2025-01-10T07:53:50,918 DEBUG [Thread-80 {}] bucket.BucketCache(1619): Check for key -8040123348610602668_0 failed. Evicting. 2025-01-10T07:53:50,918 DEBUG [Thread-80 {}] bucket.BucketCache(843): removed key -8040123348610602668_0 from back map with offset lock 2089984 in the evict process 2025-01-10T07:53:50,919 INFO [Thread-80 {}] bucket.BucketCache(1625): Finished validating 1 keys in the backing map. Recovered: 0. This took 1ms. 2025-01-10T07:53:50,950 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:50,950 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:50,951 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8d68ec26-bb55-e9b9-9e09-997e8dcb41f6/bucket.persistence17364956279441736495630951 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:50,951 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8d68ec26-bb55-e9b9-9e09-997e8dcb41f6/bucket.persistence17364956279441736495630951 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:50,960 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:50,960 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:50,960 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d6bd01cd-a623-1d16-5ab4-57fa19bd0211/bucket.persistence1736495630960 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:50,960 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d6bd01cd-a623-1d16-5ab4-57fa19bd0211/bucket.persistence1736495630960 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:50,979 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [1,000] milli-secs(wait.for.ratio=[1]) 2025-01-10T07:53:51,055 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:51,056 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:51,056 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8d68ec26-bb55-e9b9-9e09-997e8dcb41f6/bucket.persistence17364956279441736495631056 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:51,056 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8d68ec26-bb55-e9b9-9e09-997e8dcb41f6/bucket.persistence17364956279441736495631056 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:51,080 INFO [Time-limited test {}] bucket.BucketCache(1726): Disabling cache 2025-01-10T07:53:51,080 INFO [Time-limited test {}] bucket.BucketCache(1750): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a3db39a4-a386-2af8-3c1e-c42d5c261030/bucket.persistence 2025-01-10T07:53:51,080 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2025-01-10T07:53:51,080 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2025-01-10T07:53:51,081 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2025-01-10T07:53:51,081 INFO [Time-limited test {}] bucket.BucketCache(1756): Shutting down cache persister thread. 2025-01-10T07:53:51,081 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2025-01-10T07:53:51,091 DEBUG [Time-limited test {}] bucket.BucketCache(1474): Thread Time-limited test started persisting bucket cache to file 2025-01-10T07:53:51,091 DEBUG [Time-limited test {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:51,091 DEBUG [Time-limited test {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:51,096 DEBUG [Time-limited test {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2025-01-10T07:53:51,096 DEBUG [Time-limited test {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1736495631091 2025-01-10T07:53:51,096 DEBUG [Time-limited test {}] bucket.BucketCache(1496): Thread Time-limited test finished persisting bucket cache to file, renaming 2025-01-10T07:53:51,118 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a3db39a4-a386-2af8-3c1e-c42d5c261030/bucket.cache 2025-01-10T07:53:51,118 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:51,118 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:51,118 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2025-01-10T07:53:51,119 INFO [Thread-82 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:51,119 WARN [Thread-82 {}] bucket.BucketCache(1520): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2025-01-10T07:53:51,119 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a3db39a4-a386-2af8-3c1e-c42d5c261030/bucket.cache, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a3db39a4-a386-2af8-3c1e-c42d5c261030/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:51,119 INFO [Thread-82 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:51,119 INFO [Thread-82 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a3db39a4-a386-2af8-3c1e-c42d5c261030/bucket.persistence is complete. 2025-01-10T07:53:51,187 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:51,187 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:51,187 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:51,187 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:51,191 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:51,191 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956258571736495631187 2025-01-10T07:53:51,191 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:51,229 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestVerifyBucketCacheFile#testRetrieveFromFile[1: blockSize=16,384, bucketSizes=[I@388526fb] Thread=88 (was 83) - Thread LEAK? -, OpenFileDescriptor=301 (was 300) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=495 (was 495), ProcessCount=11 (was 11), AvailableMemoryMB=6309 (was 6239) - AvailableMemoryMB LEAK? - 2025-01-10T07:53:51,237 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestVerifyBucketCacheFile#testModifiedBucketCacheFileData[1: blockSize=16,384, bucketSizes=[I@388526fb] Thread=88, OpenFileDescriptor=301, MaxFileDescriptor=1048576, SystemLoadAverage=495, ProcessCount=11, AvailableMemoryMB=6308 2025-01-10T07:53:51,262 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-log-dir so I do NOT create it in target/test-data/6c7aacbd-85d1-cbca-703a-e77e5e7d1c22 2025-01-10T07:53:51,262 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.log.dir property value differs in configuration and system: Configuration=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/../logs while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-log-dir Erasing configuration value by system value. 2025-01-10T07:53:51,262 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-tmp-dir so I do NOT create it in target/test-data/6c7aacbd-85d1-cbca-703a-e77e5e7d1c22 2025-01-10T07:53:51,262 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-tmp-dir Erasing configuration value by system value. 2025-01-10T07:53:51,263 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/6c7aacbd-85d1-cbca-703a-e77e5e7d1c22 2025-01-10T07:53:51,263 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-10T07:53:51,286 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/6c7aacbd-85d1-cbca-703a-e77e5e7d1c22/bucket.cache 2025-01-10T07:53:51,287 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:51,287 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:51,287 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 9223372036854775807 2025-01-10T07:53:51,288 INFO [Thread-85 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:51,288 WARN [Thread-85 {}] bucket.BucketCache(1520): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2025-01-10T07:53:51,288 INFO [Thread-85 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:51,288 INFO [Thread-85 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/6c7aacbd-85d1-cbca-703a-e77e5e7d1c22/bucket.persistence is complete. 2025-01-10T07:53:51,289 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/6c7aacbd-85d1-cbca-703a-e77e5e7d1c22/bucket.cache, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/6c7aacbd-85d1-cbca-703a-e77e5e7d1c22/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:51,345 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:51,345 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:51,345 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7710cb5e-4abc-151f-5509-17a961013923/bucket.persistence1736495631345 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:51,346 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7710cb5e-4abc-151f-5509-17a961013923/bucket.persistence1736495631345 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:51,389 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:51,390 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:51,390 INFO [Time-limited test {}] bucket.BucketCache(1726): Disabling cache 2025-01-10T07:53:51,390 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/81823344-9ae7-f60d-9761-52816b12ba6b/bucket.persistence17364956263811736495631390 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:51,390 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/81823344-9ae7-f60d-9761-52816b12ba6b/bucket.persistence17364956263811736495631390 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:51,391 INFO [Time-limited test {}] bucket.BucketCache(1750): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/6c7aacbd-85d1-cbca-703a-e77e5e7d1c22/bucket.persistence 2025-01-10T07:53:51,391 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2025-01-10T07:53:51,391 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2025-01-10T07:53:51,391 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2025-01-10T07:53:51,391 INFO [Time-limited test {}] bucket.BucketCache(1756): Shutting down cache persister thread. 2025-01-10T07:53:51,391 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2025-01-10T07:53:51,401 DEBUG [Time-limited test {}] bucket.BucketCache(1474): Thread Time-limited test started persisting bucket cache to file 2025-01-10T07:53:51,402 DEBUG [Time-limited test {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:51,402 DEBUG [Time-limited test {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:51,406 DEBUG [Time-limited test {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2025-01-10T07:53:51,406 DEBUG [Time-limited test {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1736495631401 2025-01-10T07:53:51,406 DEBUG [Time-limited test {}] bucket.BucketCache(1496): Thread Time-limited test finished persisting bucket cache to file, renaming 2025-01-10T07:53:51,426 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/6c7aacbd-85d1-cbca-703a-e77e5e7d1c22/bucket.cache 2025-01-10T07:53:51,427 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:51,427 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:51,427 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2025-01-10T07:53:51,428 INFO [Thread-87 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:51,428 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/6c7aacbd-85d1-cbca-703a-e77e5e7d1c22/bucket.cache, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/6c7aacbd-85d1-cbca-703a-e77e5e7d1c22/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:51,428 INFO [Thread-87 {}] bucket.BucketCache(1540): Reading new chunked format of persistence. 2025-01-10T07:53:51,428 INFO [Thread-87 {}] bucket.BucketCache(1693): Retrieved 1 of chunks with blockCount = 1. 2025-01-10T07:53:51,433 WARN [Thread-87 {}] bucket.BucketCache(1601): Checksum for cache file failed. We need to validate each cache key in the backing map. This may take some time, so we'll do it in a background thread, 2025-01-10T07:53:51,434 DEBUG [Thread-87 {}] bucket.BucketCache(1557): Updating region size map after retrieving cached file list 2025-01-10T07:53:51,434 INFO [Thread-87 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:51,434 INFO [Thread-87 {}] bucket.BucketCache(1550): Bucket cache retrieved from file successfully with size: 1 2025-01-10T07:53:51,434 INFO [Thread-87 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/6c7aacbd-85d1-cbca-703a-e77e5e7d1c22/bucket.persistence is complete. 2025-01-10T07:53:51,484 DEBUG [Thread-89 {}] bucket.BucketCache(1619): Check for key 8754328773968203466_0 failed. Evicting. 2025-01-10T07:53:51,484 DEBUG [Thread-89 {}] bucket.BucketCache(843): removed key 8754328773968203466_0 from back map with offset lock 2089984 in the evict process 2025-01-10T07:53:51,485 INFO [Thread-89 {}] bucket.BucketCache(1625): Finished validating 1 keys in the backing map. Recovered: 0. This took 1ms. 2025-01-10T07:53:51,491 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:51,492 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:51,492 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:51,492 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:51,496 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:51,496 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956258571736495631492 2025-01-10T07:53:51,496 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:51,538 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestVerifyBucketCacheFile#testModifiedBucketCacheFileData[1: blockSize=16,384, bucketSizes=[I@388526fb] Thread=93 (was 88) - Thread LEAK? -, OpenFileDescriptor=302 (was 301) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=495 (was 495), ProcessCount=11 (was 11), AvailableMemoryMB=6291 (was 6308) 2025-01-10T07:53:51,546 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestVerifyBucketCacheFile#testModifiedBucketCacheFileTime[1: blockSize=16,384, bucketSizes=[I@388526fb] Thread=93, OpenFileDescriptor=302, MaxFileDescriptor=1048576, SystemLoadAverage=495, ProcessCount=11, AvailableMemoryMB=6290 2025-01-10T07:53:51,559 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-log-dir so I do NOT create it in target/test-data/fd224a2c-fb56-5601-4369-7869a19fed6b 2025-01-10T07:53:51,559 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.log.dir property value differs in configuration and system: Configuration=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/../logs while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-log-dir Erasing configuration value by system value. 2025-01-10T07:53:51,559 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-tmp-dir so I do NOT create it in target/test-data/fd224a2c-fb56-5601-4369-7869a19fed6b 2025-01-10T07:53:51,559 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-tmp-dir Erasing configuration value by system value. 2025-01-10T07:53:51,559 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/fd224a2c-fb56-5601-4369-7869a19fed6b 2025-01-10T07:53:51,559 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-10T07:53:51,576 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/fd224a2c-fb56-5601-4369-7869a19fed6b/bucket.cache 2025-01-10T07:53:51,576 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:51,576 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:51,576 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2025-01-10T07:53:51,576 INFO [Thread-92 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:51,576 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/fd224a2c-fb56-5601-4369-7869a19fed6b/bucket.cache, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/fd224a2c-fb56-5601-4369-7869a19fed6b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:51,577 WARN [Thread-92 {}] bucket.BucketCache(1520): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2025-01-10T07:53:51,577 INFO [Thread-92 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:51,577 INFO [Thread-92 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/fd224a2c-fb56-5601-4369-7869a19fed6b/bucket.persistence is complete. 2025-01-10T07:53:51,722 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:51,722 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:51,722 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d7fb7f24-0a10-4128-c40c-2ca16622cf4e/bucket.persistence17364956292061736495631722 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:51,722 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d7fb7f24-0a10-4128-c40c-2ca16622cf4e/bucket.persistence17364956292061736495631722 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:51,727 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:51,727 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:51,727 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/088e174c-7fea-2eeb-ebfa-5834b72fd3db/bucket.persistence1736495631727 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:51,727 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/088e174c-7fea-2eeb-ebfa-5834b72fd3db/bucket.persistence1736495631727 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:51,777 INFO [Time-limited test {}] bucket.BucketCache(1726): Disabling cache 2025-01-10T07:53:51,777 INFO [Time-limited test {}] bucket.BucketCache(1750): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/fd224a2c-fb56-5601-4369-7869a19fed6b/bucket.persistence 2025-01-10T07:53:51,777 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2025-01-10T07:53:51,777 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2025-01-10T07:53:51,777 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2025-01-10T07:53:51,778 INFO [Time-limited test {}] bucket.BucketCache(1756): Shutting down cache persister thread. 2025-01-10T07:53:51,778 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2025-01-10T07:53:51,788 DEBUG [Time-limited test {}] bucket.BucketCache(1474): Thread Time-limited test started persisting bucket cache to file 2025-01-10T07:53:51,788 DEBUG [Time-limited test {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:51,788 DEBUG [Time-limited test {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:51,792 DEBUG [Time-limited test {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2025-01-10T07:53:51,792 DEBUG [Time-limited test {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1736495631788 2025-01-10T07:53:51,792 DEBUG [Time-limited test {}] bucket.BucketCache(1496): Thread Time-limited test finished persisting bucket cache to file, renaming 2025-01-10T07:53:51,795 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:51,795 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:51,795 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/81823344-9ae7-f60d-9761-52816b12ba6b/bucket.persistence17364956263811736495631795 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:51,795 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/81823344-9ae7-f60d-9761-52816b12ba6b/bucket.persistence17364956263811736495631795 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:51,797 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:51,797 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:51,797 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:51,797 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:51,806 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:51,806 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956258571736495631797 2025-01-10T07:53:51,806 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:51,810 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/fd224a2c-fb56-5601-4369-7869a19fed6b/bucket.cache 2025-01-10T07:53:51,810 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:51,810 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:51,810 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2025-01-10T07:53:51,811 INFO [Thread-95 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:51,811 INFO [Thread-95 {}] bucket.BucketCache(1540): Reading new chunked format of persistence. 2025-01-10T07:53:51,811 INFO [Thread-95 {}] bucket.BucketCache(1693): Retrieved 1 of chunks with blockCount = 1. 2025-01-10T07:53:51,811 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/fd224a2c-fb56-5601-4369-7869a19fed6b/bucket.cache, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/fd224a2c-fb56-5601-4369-7869a19fed6b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:51,814 WARN [Thread-95 {}] bucket.BucketCache(1601): Checksum for cache file failed. We need to validate each cache key in the backing map. This may take some time, so we'll do it in a background thread, 2025-01-10T07:53:51,817 DEBUG [Thread-95 {}] bucket.BucketCache(1557): Updating region size map after retrieving cached file list 2025-01-10T07:53:51,817 INFO [Thread-95 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:51,818 INFO [Thread-95 {}] bucket.BucketCache(1550): Bucket cache retrieved from file successfully with size: 1 2025-01-10T07:53:51,818 INFO [Thread-95 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/fd224a2c-fb56-5601-4369-7869a19fed6b/bucket.persistence is complete. 2025-01-10T07:53:51,868 INFO [Thread-97 {}] bucket.BucketCache(1625): Finished validating 1 keys in the backing map. Recovered: 1. This took 1ms. 2025-01-10T07:53:51,921 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestVerifyBucketCacheFile#testModifiedBucketCacheFileTime[1: blockSize=16,384, bucketSizes=[I@388526fb] Thread=98 (was 93) - Thread LEAK? -, OpenFileDescriptor=303 (was 302) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=495 (was 495), ProcessCount=11 (was 11), AvailableMemoryMB=6228 (was 6290) 2025-01-10T07:53:51,932 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestVerifyBucketCacheFile#testRetrieveFromFileAfterDelete[1: blockSize=16,384, bucketSizes=[I@388526fb] Thread=98, OpenFileDescriptor=303, MaxFileDescriptor=1048576, SystemLoadAverage=495, ProcessCount=11, AvailableMemoryMB=6228 2025-01-10T07:53:51,954 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-log-dir so I do NOT create it in target/test-data/4c1f8844-59fd-3d74-ac37-811c3668bba4 2025-01-10T07:53:51,955 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.log.dir property value differs in configuration and system: Configuration=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/../logs while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-log-dir Erasing configuration value by system value. 2025-01-10T07:53:51,955 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-tmp-dir so I do NOT create it in target/test-data/4c1f8844-59fd-3d74-ac37-811c3668bba4 2025-01-10T07:53:51,955 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-tmp-dir Erasing configuration value by system value. 2025-01-10T07:53:51,955 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/4c1f8844-59fd-3d74-ac37-811c3668bba4 2025-01-10T07:53:51,955 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-10T07:53:51,960 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/4c1f8844-59fd-3d74-ac37-811c3668bba4/bucket.cache 2025-01-10T07:53:51,960 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:51,960 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:51,960 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 300 2025-01-10T07:53:51,962 INFO [Thread-99 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:51,962 WARN [Thread-99 {}] bucket.BucketCache(1520): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2025-01-10T07:53:51,962 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/4c1f8844-59fd-3d74-ac37-811c3668bba4/bucket.cache, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/4c1f8844-59fd-3d74-ac37-811c3668bba4/bucket.persistence1736495631959, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:51,962 INFO [Thread-99 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:51,962 INFO [Thread-99 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/4c1f8844-59fd-3d74-ac37-811c3668bba4/bucket.persistence1736495631959 is complete. 2025-01-10T07:53:52,106 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:52,107 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:52,107 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:52,107 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:52,111 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:52,111 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956258571736495632107 2025-01-10T07:53:52,111 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:52,163 INFO [Time-limited test {}] bucket.BucketCache(1726): Disabling cache 2025-01-10T07:53:52,163 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2025-01-10T07:53:52,163 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2025-01-10T07:53:52,163 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1258): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2025-01-10T07:53:52,164 INFO [Time-limited test {}] bucket.BucketCache(1750): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/4c1f8844-59fd-3d74-ac37-811c3668bba4/bucket.persistence1736495631959 2025-01-10T07:53:52,164 INFO [Time-limited test {}] bucket.BucketCache(1756): Shutting down cache persister thread. 2025-01-10T07:53:52,164 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2025-01-10T07:53:52,174 DEBUG [Time-limited test {}] bucket.BucketCache(1474): Thread Time-limited test started persisting bucket cache to file 2025-01-10T07:53:52,174 DEBUG [Time-limited test {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:52,174 DEBUG [Time-limited test {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:52,179 DEBUG [Time-limited test {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2025-01-10T07:53:52,179 DEBUG [Time-limited test {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence17364956319591736495632174 2025-01-10T07:53:52,179 DEBUG [Time-limited test {}] bucket.BucketCache(1496): Thread Time-limited test finished persisting bucket cache to file, renaming 2025-01-10T07:53:52,530 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/4c1f8844-59fd-3d74-ac37-811c3668bba4/bucket.cache 2025-01-10T07:53:52,530 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:52,530 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:52,530 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 300 2025-01-10T07:53:52,531 INFO [Thread-102 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:52,531 WARN [Thread-102 {}] bucket.BucketCache(1520): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2025-01-10T07:53:52,531 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/4c1f8844-59fd-3d74-ac37-811c3668bba4/bucket.cache, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/4c1f8844-59fd-3d74-ac37-811c3668bba4/bucket.persistence1736495631959, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:52,531 INFO [Thread-102 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:52,531 INFO [Thread-102 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/4c1f8844-59fd-3d74-ac37-811c3668bba4/bucket.persistence1736495631959 is complete. 2025-01-10T07:53:52,641 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestVerifyBucketCacheFile#testRetrieveFromFileAfterDelete[1: blockSize=16,384, bucketSizes=[I@388526fb] Thread=103 (was 98) - Thread LEAK? -, OpenFileDescriptor=304 (was 303) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=495 (was 495), ProcessCount=11 (was 11), AvailableMemoryMB=6203 (was 6228) 2025-01-10T07:53:52,650 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestVerifyBucketCacheFile#testPartiallyFilledChunks[1: blockSize=16,384, bucketSizes=[I@388526fb] Thread=103, OpenFileDescriptor=304, MaxFileDescriptor=1048576, SystemLoadAverage=495, ProcessCount=11, AvailableMemoryMB=6203 2025-01-10T07:53:52,670 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-log-dir so I do NOT create it in target/test-data/698a1ce4-f8b7-d3db-e484-5bf666cdb28b 2025-01-10T07:53:52,671 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.log.dir property value differs in configuration and system: Configuration=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/../logs while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-log-dir Erasing configuration value by system value. 2025-01-10T07:53:52,671 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-tmp-dir so I do NOT create it in target/test-data/698a1ce4-f8b7-d3db-e484-5bf666cdb28b 2025-01-10T07:53:52,671 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-tmp-dir Erasing configuration value by system value. 2025-01-10T07:53:52,671 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/698a1ce4-f8b7-d3db-e484-5bf666cdb28b 2025-01-10T07:53:52,671 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-10T07:53:52,698 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/698a1ce4-f8b7-d3db-e484-5bf666cdb28b/bucket.cache 2025-01-10T07:53:52,699 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:52,699 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:52,699 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2025-01-10T07:53:52,706 INFO [Thread-104 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:52,706 WARN [Thread-104 {}] bucket.BucketCache(1520): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2025-01-10T07:53:52,706 INFO [Thread-104 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:52,706 INFO [Thread-104 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/698a1ce4-f8b7-d3db-e484-5bf666cdb28b/bucket.persistence1736495632698 is complete. 2025-01-10T07:53:52,712 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/698a1ce4-f8b7-d3db-e484-5bf666cdb28b/bucket.cache, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/698a1ce4-f8b7-d3db-e484-5bf666cdb28b/bucket.persistence1736495632698, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:52,796 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:52,796 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:52,796 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/81823344-9ae7-f60d-9761-52816b12ba6b/bucket.persistence17364956263811736495632796 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:52,796 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/81823344-9ae7-f60d-9761-52816b12ba6b/bucket.persistence17364956263811736495632796 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:52,811 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:52,811 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:52,811 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/fd224a2c-fb56-5601-4369-7869a19fed6b/bucket.persistence1736495632811 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:52,812 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/fd224a2c-fb56-5601-4369-7869a19fed6b/bucket.persistence1736495632811 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:52,830 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:52,831 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:52,831 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:52,831 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:52,842 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:52,842 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956319591736495632831 2025-01-10T07:53:52,843 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:52,951 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:52,952 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:52,952 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8d68ec26-bb55-e9b9-9e09-997e8dcb41f6/bucket.persistence17364956279441736495632952 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:52,952 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8d68ec26-bb55-e9b9-9e09-997e8dcb41f6/bucket.persistence17364956279441736495632952 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:52,961 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:52,961 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:52,961 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d6bd01cd-a623-1d16-5ab4-57fa19bd0211/bucket.persistence1736495632961 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:52,961 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d6bd01cd-a623-1d16-5ab4-57fa19bd0211/bucket.persistence1736495632961 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:53,013 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:53,014 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:53,014 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:53,014 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:53,019 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:53,019 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956258571736495633014 2025-01-10T07:53:53,019 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:53,119 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:53,119 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:53,119 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a3db39a4-a386-2af8-3c1e-c42d5c261030/bucket.persistence1736495633119 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:53,120 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a3db39a4-a386-2af8-3c1e-c42d5c261030/bucket.persistence1736495633119 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:53,143 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:53,143 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:53,143 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:53,143 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:53,150 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:53,151 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956319591736495633143 2025-01-10T07:53:53,151 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:53,320 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:53,320 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:53,320 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:53,320 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:53,327 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:53,327 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956258571736495633320 2025-01-10T07:53:53,327 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:53,428 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:53,428 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:53,428 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/6c7aacbd-85d1-cbca-703a-e77e5e7d1c22/bucket.persistence1736495633428 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:53,428 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/6c7aacbd-85d1-cbca-703a-e77e5e7d1c22/bucket.persistence1736495633428 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:53,451 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:53,451 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:53,451 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:53,451 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:53,455 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:53,455 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956319591736495633451 2025-01-10T07:53:53,455 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:53,627 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:53,627 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:53,628 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:53,628 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:53,633 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:53,633 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956258571736495633627 2025-01-10T07:53:53,633 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:53,726 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:53,726 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:53,726 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d7fb7f24-0a10-4128-c40c-2ca16622cf4e/bucket.persistence17364956292061736495633726 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:53,726 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d7fb7f24-0a10-4128-c40c-2ca16622cf4e/bucket.persistence17364956292061736495633726 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:53,728 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:53,728 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:53,728 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/088e174c-7fea-2eeb-ebfa-5834b72fd3db/bucket.persistence1736495633728 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:53,728 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/088e174c-7fea-2eeb-ebfa-5834b72fd3db/bucket.persistence1736495633728 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:53,756 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:53,756 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:53,756 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:53,756 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:53,762 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:53,762 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956319591736495633756 2025-01-10T07:53:53,762 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:53,934 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:53,934 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:53,934 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:53,934 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:53,938 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:53,939 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956258571736495633934 2025-01-10T07:53:53,939 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:54,016 DEBUG [Time-limited test {}] bucket.BucketCache(1474): Thread Time-limited test started persisting bucket cache to file 2025-01-10T07:53:54,017 DEBUG [Time-limited test {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:54,017 DEBUG [Time-limited test {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 13, fullycachedFiles size: 0, chunkSize: 5 2025-01-10T07:53:54,022 DEBUG [Time-limited test {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 13, fullycachedFiles size: 0 2025-01-10T07:53:54,022 DEBUG [Time-limited test {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 13, fullycachedFiles size: 0, file name: bucket.persistence17364956326981736495634016 2025-01-10T07:53:54,022 DEBUG [Time-limited test {}] bucket.BucketCache(1496): Thread Time-limited test finished persisting bucket cache to file, renaming 2025-01-10T07:53:54,022 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/698a1ce4-f8b7-d3db-e484-5bf666cdb28b/bucket.cache 2025-01-10T07:53:54,022 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:54,023 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:54,023 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2025-01-10T07:53:54,025 INFO [Thread-114 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:54,025 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/698a1ce4-f8b7-d3db-e484-5bf666cdb28b/bucket.cache, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/698a1ce4-f8b7-d3db-e484-5bf666cdb28b/bucket.persistence1736495632698, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:54,025 INFO [Thread-114 {}] bucket.BucketCache(1540): Reading new chunked format of persistence. 2025-01-10T07:53:54,027 INFO [Thread-114 {}] bucket.BucketCache(1693): Retrieved 3 of chunks with blockCount = 13. 2025-01-10T07:53:54,034 DEBUG [Thread-114 {}] bucket.BucketCache(1557): Updating region size map after retrieving cached file list 2025-01-10T07:53:54,034 INFO [Thread-114 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:54,034 INFO [Thread-114 {}] bucket.BucketCache(1550): Bucket cache retrieved from file successfully with size: 13 2025-01-10T07:53:54,034 INFO [Thread-114 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/698a1ce4-f8b7-d3db-e484-5bf666cdb28b/bucket.persistence1736495632698 is complete. 2025-01-10T07:53:54,138 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestVerifyBucketCacheFile#testPartiallyFilledChunks[1: blockSize=16,384, bucketSizes=[I@388526fb] Thread=113 (was 103) - Thread LEAK? -, OpenFileDescriptor=306 (was 304) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=495 (was 495), ProcessCount=11 (was 11), AvailableMemoryMB=6178 (was 6203) 2025-01-10T07:53:54,146 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestVerifyBucketCacheFile#testCompletelyFilledChunks[1: blockSize=16,384, bucketSizes=[I@388526fb] Thread=113, OpenFileDescriptor=306, MaxFileDescriptor=1048576, SystemLoadAverage=495, ProcessCount=11, AvailableMemoryMB=6177 2025-01-10T07:53:54,161 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-log-dir so I do NOT create it in target/test-data/472a6bc1-db6a-9d78-f9fc-95814d814dc2 2025-01-10T07:53:54,161 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.log.dir property value differs in configuration and system: Configuration=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/../logs while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-log-dir Erasing configuration value by system value. 2025-01-10T07:53:54,161 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-tmp-dir so I do NOT create it in target/test-data/472a6bc1-db6a-9d78-f9fc-95814d814dc2 2025-01-10T07:53:54,161 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-tmp-dir Erasing configuration value by system value. 2025-01-10T07:53:54,161 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/472a6bc1-db6a-9d78-f9fc-95814d814dc2 2025-01-10T07:53:54,161 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-10T07:53:54,184 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/472a6bc1-db6a-9d78-f9fc-95814d814dc2/bucket.cache 2025-01-10T07:53:54,184 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:54,184 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:54,184 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2025-01-10T07:53:54,186 INFO [Thread-117 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:54,186 WARN [Thread-117 {}] bucket.BucketCache(1520): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2025-01-10T07:53:54,186 INFO [Thread-117 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:54,187 INFO [Thread-117 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/472a6bc1-db6a-9d78-f9fc-95814d814dc2/bucket.persistence1736495634183 is complete. 2025-01-10T07:53:54,190 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/472a6bc1-db6a-9d78-f9fc-95814d814dc2/bucket.cache, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/472a6bc1-db6a-9d78-f9fc-95814d814dc2/bucket.persistence1736495634183, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:54,209 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:54,209 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:54,209 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d7fb7f24-0a10-4128-c40c-2ca16622cf4e/bucket.persistence17364956292061736495634209 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:54,209 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d7fb7f24-0a10-4128-c40c-2ca16622cf4e/bucket.persistence17364956292061736495634209 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:54,239 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:54,240 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:54,240 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:54,240 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:54,246 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:54,247 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956258571736495634240 2025-01-10T07:53:54,247 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:54,346 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:54,346 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:54,346 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7710cb5e-4abc-151f-5509-17a961013923/bucket.persistence1736495634346 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:54,347 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7710cb5e-4abc-151f-5509-17a961013923/bucket.persistence1736495634346 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:54,363 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:54,363 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:54,363 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:54,363 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:54,367 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:54,367 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956319591736495634363 2025-01-10T07:53:54,368 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:54,429 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:54,429 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:54,429 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/6c7aacbd-85d1-cbca-703a-e77e5e7d1c22/bucket.persistence1736495634429 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:54,429 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/6c7aacbd-85d1-cbca-703a-e77e5e7d1c22/bucket.persistence1736495634429 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:54,547 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:54,547 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:54,547 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:54,547 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:54,552 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:54,552 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956258571736495634547 2025-01-10T07:53:54,552 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:54,668 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:54,668 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:54,668 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:54,668 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:54,672 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:54,672 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956319591736495634668 2025-01-10T07:53:54,672 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:54,699 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:54,699 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:54,700 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/698a1ce4-f8b7-d3db-e484-5bf666cdb28b/bucket.persistence17364956326981736495634700 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:54,700 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/698a1ce4-f8b7-d3db-e484-5bf666cdb28b/bucket.persistence17364956326981736495634700 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:54,727 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:54,727 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:54,727 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d7fb7f24-0a10-4128-c40c-2ca16622cf4e/bucket.persistence17364956292061736495634727 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:54,727 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d7fb7f24-0a10-4128-c40c-2ca16622cf4e/bucket.persistence17364956292061736495634727 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:54,728 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:54,728 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:54,728 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/088e174c-7fea-2eeb-ebfa-5834b72fd3db/bucket.persistence1736495634728 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:54,728 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/088e174c-7fea-2eeb-ebfa-5834b72fd3db/bucket.persistence1736495634728 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:54,797 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:54,797 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:54,797 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/81823344-9ae7-f60d-9761-52816b12ba6b/bucket.persistence17364956263811736495634797 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:54,797 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/81823344-9ae7-f60d-9761-52816b12ba6b/bucket.persistence17364956263811736495634797 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:54,812 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:54,812 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:54,812 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/fd224a2c-fb56-5601-4369-7869a19fed6b/bucket.persistence1736495634812 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:54,812 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/fd224a2c-fb56-5601-4369-7869a19fed6b/bucket.persistence1736495634812 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:54,853 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:54,853 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:54,853 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:54,853 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:54,857 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:54,857 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956258571736495634853 2025-01-10T07:53:54,857 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:54,952 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:54,953 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:54,953 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8d68ec26-bb55-e9b9-9e09-997e8dcb41f6/bucket.persistence17364956279441736495634953 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:54,953 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8d68ec26-bb55-e9b9-9e09-997e8dcb41f6/bucket.persistence17364956279441736495634953 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:54,962 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:54,962 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:54,962 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d6bd01cd-a623-1d16-5ab4-57fa19bd0211/bucket.persistence1736495634962 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:54,962 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d6bd01cd-a623-1d16-5ab4-57fa19bd0211/bucket.persistence1736495634962 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:54,972 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:54,972 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:54,972 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:54,972 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:54,977 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:54,977 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956319591736495634972 2025-01-10T07:53:54,977 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:55,025 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:55,025 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:55,025 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/698a1ce4-f8b7-d3db-e484-5bf666cdb28b/bucket.persistence17364956326981736495635025 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:55,026 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/698a1ce4-f8b7-d3db-e484-5bf666cdb28b/bucket.persistence17364956326981736495635025 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:55,057 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:55,057 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:55,057 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8d68ec26-bb55-e9b9-9e09-997e8dcb41f6/bucket.persistence17364956279441736495635057 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:55,057 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/8d68ec26-bb55-e9b9-9e09-997e8dcb41f6/bucket.persistence17364956279441736495635057 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:55,120 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:55,120 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:55,120 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a3db39a4-a386-2af8-3c1e-c42d5c261030/bucket.persistence1736495635120 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:55,120 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a3db39a4-a386-2af8-3c1e-c42d5c261030/bucket.persistence1736495635120 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:55,158 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:55,158 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:55,158 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:55,158 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:55,163 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:55,163 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956258571736495635158 2025-01-10T07:53:55,163 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:55,192 DEBUG [Time-limited test {}] bucket.BucketCache(1474): Thread Time-limited test started persisting bucket cache to file 2025-01-10T07:53:55,192 DEBUG [Time-limited test {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:55,193 DEBUG [Time-limited test {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 10, fullycachedFiles size: 0, chunkSize: 5 2025-01-10T07:53:55,196 DEBUG [Time-limited test {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 10, fullycachedFiles size: 0 2025-01-10T07:53:55,196 DEBUG [Time-limited test {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 10, fullycachedFiles size: 0, file name: bucket.persistence17364956341831736495635192 2025-01-10T07:53:55,196 DEBUG [Time-limited test {}] bucket.BucketCache(1496): Thread Time-limited test finished persisting bucket cache to file, renaming 2025-01-10T07:53:55,197 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/472a6bc1-db6a-9d78-f9fc-95814d814dc2/bucket.cache 2025-01-10T07:53:55,197 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:55,197 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:55,197 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2025-01-10T07:53:55,198 INFO [Thread-126 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:55,198 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/472a6bc1-db6a-9d78-f9fc-95814d814dc2/bucket.cache, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/472a6bc1-db6a-9d78-f9fc-95814d814dc2/bucket.persistence1736495634183, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:55,198 INFO [Thread-126 {}] bucket.BucketCache(1540): Reading new chunked format of persistence. 2025-01-10T07:53:55,199 INFO [Thread-126 {}] bucket.BucketCache(1693): Retrieved 2 of chunks with blockCount = 10. 2025-01-10T07:53:55,202 DEBUG [Thread-126 {}] bucket.BucketCache(1557): Updating region size map after retrieving cached file list 2025-01-10T07:53:55,202 INFO [Thread-126 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:55,202 INFO [Thread-126 {}] bucket.BucketCache(1550): Bucket cache retrieved from file successfully with size: 10 2025-01-10T07:53:55,202 INFO [Thread-126 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/472a6bc1-db6a-9d78-f9fc-95814d814dc2/bucket.persistence1736495634183 is complete. 2025-01-10T07:53:55,308 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestVerifyBucketCacheFile#testCompletelyFilledChunks[1: blockSize=16,384, bucketSizes=[I@388526fb] Thread=123 (was 113) - Thread LEAK? -, OpenFileDescriptor=308 (was 306) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=488 (was 495), ProcessCount=11 (was 11), AvailableMemoryMB=6120 (was 6177) 2025-01-10T07:53:55,315 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestVerifyBucketCacheFile#testSingleChunk[1: blockSize=16,384, bucketSizes=[I@388526fb] Thread=123, OpenFileDescriptor=308, MaxFileDescriptor=1048576, SystemLoadAverage=488, ProcessCount=11, AvailableMemoryMB=6120 2025-01-10T07:53:55,329 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-log-dir so I do NOT create it in target/test-data/66dc9c63-f6c9-d12a-6933-a1ea54b61cc4 2025-01-10T07:53:55,329 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.log.dir property value differs in configuration and system: Configuration=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/../logs while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-log-dir Erasing configuration value by system value. 2025-01-10T07:53:55,329 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-tmp-dir so I do NOT create it in target/test-data/66dc9c63-f6c9-d12a-6933-a1ea54b61cc4 2025-01-10T07:53:55,329 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1d2be5be-bc22-df4d-1655-f2003133a1ab/hadoop-tmp-dir Erasing configuration value by system value. 2025-01-10T07:53:55,329 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/66dc9c63-f6c9-d12a-6933-a1ea54b61cc4 2025-01-10T07:53:55,329 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2025-01-10T07:53:55,345 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/66dc9c63-f6c9-d12a-6933-a1ea54b61cc4/bucket.cache 2025-01-10T07:53:55,345 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:55,345 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:55,345 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2025-01-10T07:53:55,346 INFO [Thread-129 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:55,346 WARN [Thread-129 {}] bucket.BucketCache(1520): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2025-01-10T07:53:55,346 INFO [Thread-129 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:55,346 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/66dc9c63-f6c9-d12a-6933-a1ea54b61cc4/bucket.cache, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/66dc9c63-f6c9-d12a-6933-a1ea54b61cc4/bucket.persistence1736495635345, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:55,346 INFO [Thread-129 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/66dc9c63-f6c9-d12a-6933-a1ea54b61cc4/bucket.persistence1736495635345 is complete. 2025-01-10T07:53:55,463 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:55,463 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:55,463 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:55,463 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:55,467 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:55,467 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956258571736495635463 2025-01-10T07:53:55,467 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:55,578 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:55,578 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:55,578 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:55,578 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:55,581 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:55,581 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956319591736495635578 2025-01-10T07:53:55,581 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:55,700 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:55,700 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:55,700 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/698a1ce4-f8b7-d3db-e484-5bf666cdb28b/bucket.persistence17364956326981736495635700 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:55,701 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/698a1ce4-f8b7-d3db-e484-5bf666cdb28b/bucket.persistence17364956326981736495635700 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:55,727 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:55,728 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:55,728 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d7fb7f24-0a10-4128-c40c-2ca16622cf4e/bucket.persistence17364956292061736495635728 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:55,728 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d7fb7f24-0a10-4128-c40c-2ca16622cf4e/bucket.persistence17364956292061736495635728 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:55,729 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:55,729 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:55,729 ERROR [bucket-cache-persister {}] bucket.BucketCache(1490): Failed to persist bucket cache to file java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/088e174c-7fea-2eeb-ebfa-5834b72fd3db/bucket.persistence1736495635729 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:55,729 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(59): Exception in BucketCachePersister. java.io.FileNotFoundException: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/088e174c-7fea-2eeb-ebfa-5834b72fd3db/bucket.persistence1736495635729 (No such file or directory) at java.io.FileOutputStream.open0(Native Method) ~[?:?] at java.io.FileOutputStream.open(FileOutputStream.java:293) ~[?:?] at java.io.FileOutputStream.(FileOutputStream.java:235) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1480) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2025-01-10T07:53:55,768 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:55,768 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:55,768 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:55,768 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:55,771 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:55,771 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956258571736495635768 2025-01-10T07:53:55,771 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:55,882 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2025-01-10T07:53:55,882 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1474): Thread bucket-cache-persister started persisting bucket cache to file 2025-01-10T07:53:55,882 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:55,882 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2025-01-10T07:53:55,885 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2025-01-10T07:53:55,885 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence17364956319591736495635882 2025-01-10T07:53:55,885 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1496): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2025-01-10T07:53:55,947 DEBUG [Time-limited test {}] bucket.BucketCache(1474): Thread Time-limited test started persisting bucket cache to file 2025-01-10T07:53:55,948 DEBUG [Time-limited test {}] bucket.BucketCache(1481): Persist in new chunked persistence format. 2025-01-10T07:53:55,948 DEBUG [Time-limited test {}] bucket.BucketCache(1661): persistToFile: before persisting backing map size: 5, fullycachedFiles size: 0, chunkSize: 5 2025-01-10T07:53:55,953 DEBUG [Time-limited test {}] bucket.BucketCache(1668): persistToFile: after persisting backing map size: 5, fullycachedFiles size: 0 2025-01-10T07:53:55,953 DEBUG [Time-limited test {}] bucket.BucketCache(1485): PersistToFile: after persisting backing map size: 5, fullycachedFiles size: 0, file name: bucket.persistence17364956353451736495635947 2025-01-10T07:53:55,953 DEBUG [Time-limited test {}] bucket.BucketCache(1496): Thread Time-limited test finished persisting bucket cache to file, renaming 2025-01-10T07:53:55,953 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/66dc9c63-f6c9-d12a-6933-a1ea54b61cc4/bucket.cache 2025-01-10T07:53:55,953 INFO [Time-limited test {}] bucket.BucketCache(352): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2025-01-10T07:53:55,953 INFO [Time-limited test {}] bucket.BucketCache(465): Starting BucketCachePersisterThread 2025-01-10T07:53:55,953 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2025-01-10T07:53:55,955 INFO [Thread-135 {}] bucket.BucketCache(1517): Started retrieving bucket cache from file 2025-01-10T07:53:55,955 INFO [Thread-135 {}] bucket.BucketCache(1540): Reading new chunked format of persistence. 2025-01-10T07:53:55,956 INFO [Thread-135 {}] bucket.BucketCache(1693): Retrieved 1 of chunks with blockCount = 5. 2025-01-10T07:53:55,966 INFO [Time-limited test {}] bucket.BucketCache(390): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/66dc9c63-f6c9-d12a-6933-a1ea54b61cc4/bucket.cache, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/66dc9c63-f6c9-d12a-6933-a1ea54b61cc4/bucket.persistence1736495635345, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2025-01-10T07:53:55,966 DEBUG [Thread-135 {}] bucket.BucketCache(1557): Updating region size map after retrieving cached file list 2025-01-10T07:53:55,967 INFO [Thread-135 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2025-01-10T07:53:55,967 INFO [Thread-135 {}] bucket.BucketCache(1550): Bucket cache retrieved from file successfully with size: 5 2025-01-10T07:53:55,967 INFO [Thread-135 {}] bucket.BucketCache(400): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/66dc9c63-f6c9-d12a-6933-a1ea54b61cc4/bucket.persistence1736495635345 is complete. 2025-01-10T07:53:56,077 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestVerifyBucketCacheFile#testSingleChunk[1: blockSize=16,384, bucketSizes=[I@388526fb] Thread=133 (was 123) - Thread LEAK? -, OpenFileDescriptor=310 (was 308) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=488 (was 488), ProcessCount=11 (was 11), AvailableMemoryMB=6115 (was 6120)